2024-12-07 00:51:14,741 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-07 00:51:14,757 main DEBUG Took 0.013583 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 00:51:14,757 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 00:51:14,757 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 00:51:14,758 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 00:51:14,759 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,766 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 00:51:14,777 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,778 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,779 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,780 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,781 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,782 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,783 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,784 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,784 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,786 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,786 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 00:51:14,787 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,787 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 00:51:14,788 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 00:51:14,790 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 00:51:14,791 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 00:51:14,791 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 00:51:14,793 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 00:51:14,793 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 00:51:14,801 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 00:51:14,803 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 00:51:14,805 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 00:51:14,805 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 00:51:14,805 main DEBUG createAppenders(={Console}) 2024-12-07 00:51:14,806 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-12-07 00:51:14,806 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-12-07 00:51:14,807 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-12-07 00:51:14,807 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 00:51:14,807 main DEBUG OutputStream closed 2024-12-07 00:51:14,808 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 00:51:14,808 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 00:51:14,808 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-12-07 00:51:14,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 00:51:14,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 00:51:14,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 00:51:14,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 00:51:14,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 00:51:14,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 00:51:14,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 00:51:14,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 00:51:14,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 00:51:14,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 00:51:14,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 00:51:14,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 00:51:14,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 00:51:14,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 00:51:14,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 00:51:14,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 00:51:14,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 00:51:14,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 00:51:14,883 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 00:51:14,883 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-12-07 00:51:14,884 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 00:51:14,884 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-12-07T00:51:14,896 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-12-07 00:51:14,898 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 00:51:14,899 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T00:51:14,942 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=8323 2024-12-07T00:51:15,133 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1 2024-12-07T00:51:15,650 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T00:51:15,739 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T00:51:15,790 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-12-07T00:51:15,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532675824 2024-12-07T00:51:15,893 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532675824 2024-12-07T00:51:15,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:15,965 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-12-07T00:51:15,965 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-12-07T00:51:16,002 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1f692feca4f5b23e8ac2ba4a607b7670, NAME => 'testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1 2024-12-07T00:51:16,046 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T00:51:16,051 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T00:51:16,133 INFO [StoreOpener-1f692feca4f5b23e8ac2ba4a607b7670-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 1f692feca4f5b23e8ac2ba4a607b7670 2024-12-07T00:51:16,171 INFO [StoreOpener-1f692feca4f5b23e8ac2ba4a607b7670-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f692feca4f5b23e8ac2ba4a607b7670 columnFamilyName MyCF 2024-12-07T00:51:16,177 DEBUG [StoreOpener-1f692feca4f5b23e8ac2ba4a607b7670-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T00:51:16,183 INFO [StoreOpener-1f692feca4f5b23e8ac2ba4a607b7670-1 {}] regionserver.HStore(327): Store=1f692feca4f5b23e8ac2ba4a607b7670/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T00:51:16,187 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f692feca4f5b23e8ac2ba4a607b7670 2024-12-07T00:51:16,192 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/data/default/testLockupAroundBadAssignSync/1f692feca4f5b23e8ac2ba4a607b7670 2024-12-07T00:51:16,193 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/data/default/testLockupAroundBadAssignSync/1f692feca4f5b23e8ac2ba4a607b7670 2024-12-07T00:51:16,195 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f692feca4f5b23e8ac2ba4a607b7670 2024-12-07T00:51:16,196 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f692feca4f5b23e8ac2ba4a607b7670 2024-12-07T00:51:16,202 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f692feca4f5b23e8ac2ba4a607b7670 2024-12-07T00:51:16,212 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/data/default/testLockupAroundBadAssignSync/1f692feca4f5b23e8ac2ba4a607b7670/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T00:51:16,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f692feca4f5b23e8ac2ba4a607b7670; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58811340, jitterRate=-0.12364274263381958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T00:51:16,220 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T00:51:16,231 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f692feca4f5b23e8ac2ba4a607b7670: Writing region info on filesystem at 1733532676094Initializing all the Stores at 1733532676095 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1733532676096 (+1 ms)Cleaning up temporary data from old regions at 1733532676197 (+101 ms)Region opened successfully at 1733532676229 (+32 ms) 2024-12-07T00:51:16,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532675824) roll requested 2024-12-07T00:51:16,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676282 2024-12-07T00:51:16,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532675824 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676282 2024-12-07T00:51:16,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676282) roll requested 2024-12-07T00:51:16,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676293 2024-12-07T00:51:16,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676293 2024-12-07T00:51:16,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676293) roll requested 2024-12-07T00:51:16,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676303 2024-12-07T00:51:16,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676303 2024-12-07T00:51:16,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676303) roll requested 2024-12-07T00:51:16,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676313 2024-12-07T00:51:16,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676313 2024-12-07T00:51:16,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676313) roll requested 2024-12-07T00:51:16,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676325 2024-12-07T00:51:16,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676325 2024-12-07T00:51:16,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676325) roll requested 2024-12-07T00:51:16,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676335 2024-12-07T00:51:16,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676335 2024-12-07T00:51:16,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676335) roll requested 2024-12-07T00:51:16,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676345 2024-12-07T00:51:16,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676345 2024-12-07T00:51:16,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676345) roll requested 2024-12-07T00:51:16,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676356 2024-12-07T00:51:16,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676356 2024-12-07T00:51:16,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676356) roll requested 2024-12-07T00:51:16,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676366 2024-12-07T00:51:16,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676366 2024-12-07T00:51:16,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676366) roll requested 2024-12-07T00:51:16,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676375 2024-12-07T00:51:16,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676375 2024-12-07T00:51:16,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676375) roll requested 2024-12-07T00:51:16,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676390 2024-12-07T00:51:16,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676390 2024-12-07T00:51:16,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676390) roll requested 2024-12-07T00:51:16,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676401 2024-12-07T00:51:16,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676401 2024-12-07T00:51:16,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676401) roll requested 2024-12-07T00:51:16,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676411 2024-12-07T00:51:16,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676411 2024-12-07T00:51:16,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676411) roll requested 2024-12-07T00:51:16,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676423 2024-12-07T00:51:16,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676423 2024-12-07T00:51:16,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676423) roll requested 2024-12-07T00:51:16,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676432 2024-12-07T00:51:16,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676432 2024-12-07T00:51:16,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676432) roll requested 2024-12-07T00:51:16,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676442 2024-12-07T00:51:16,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676442 2024-12-07T00:51:16,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676442) roll requested 2024-12-07T00:51:16,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676449 2024-12-07T00:51:16,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676449 2024-12-07T00:51:16,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676449) roll requested 2024-12-07T00:51:16,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676459 2024-12-07T00:51:16,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676459 2024-12-07T00:51:16,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676459) roll requested 2024-12-07T00:51:16,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676470 2024-12-07T00:51:16,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676470 2024-12-07T00:51:16,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676470) roll requested 2024-12-07T00:51:16,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676479 2024-12-07T00:51:16,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676479 2024-12-07T00:51:16,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676479) roll requested 2024-12-07T00:51:16,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676493 2024-12-07T00:51:16,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676493 2024-12-07T00:51:16,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676493) roll requested 2024-12-07T00:51:16,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676502 2024-12-07T00:51:16,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676502 2024-12-07T00:51:16,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676502) roll requested 2024-12-07T00:51:16,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676517 2024-12-07T00:51:16,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676517 2024-12-07T00:51:16,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676517) roll requested 2024-12-07T00:51:16,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676526 2024-12-07T00:51:16,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676526 2024-12-07T00:51:16,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676526) roll requested 2024-12-07T00:51:16,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676535 2024-12-07T00:51:16,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676535 2024-12-07T00:51:16,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676535) roll requested 2024-12-07T00:51:16,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676542 2024-12-07T00:51:16,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676542 2024-12-07T00:51:16,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676542) roll requested 2024-12-07T00:51:16,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676549 2024-12-07T00:51:16,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676549 2024-12-07T00:51:16,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676549) roll requested 2024-12-07T00:51:16,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676558 2024-12-07T00:51:16,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676558 2024-12-07T00:51:16,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676558) roll requested 2024-12-07T00:51:16,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676568 2024-12-07T00:51:16,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676568 2024-12-07T00:51:16,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676568) roll requested 2024-12-07T00:51:16,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676578 2024-12-07T00:51:16,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676578 2024-12-07T00:51:16,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676578) roll requested 2024-12-07T00:51:16,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676587 2024-12-07T00:51:16,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676587 2024-12-07T00:51:16,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676587) roll requested 2024-12-07T00:51:16,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676599 2024-12-07T00:51:16,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676599 2024-12-07T00:51:16,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676599) roll requested 2024-12-07T00:51:16,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676607 2024-12-07T00:51:16,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676607 2024-12-07T00:51:16,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676607) roll requested 2024-12-07T00:51:16,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676616 2024-12-07T00:51:16,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676616 2024-12-07T00:51:16,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676616) roll requested 2024-12-07T00:51:16,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676624 2024-12-07T00:51:16,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676624 2024-12-07T00:51:16,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676624) roll requested 2024-12-07T00:51:16,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676631 2024-12-07T00:51:16,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676631 2024-12-07T00:51:16,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676631) roll requested 2024-12-07T00:51:16,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676641 2024-12-07T00:51:16,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676641 2024-12-07T00:51:16,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676641) roll requested 2024-12-07T00:51:16,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676652 2024-12-07T00:51:16,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676652 2024-12-07T00:51:16,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676652) roll requested 2024-12-07T00:51:16,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676665 2024-12-07T00:51:16,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676665 2024-12-07T00:51:16,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676665) roll requested 2024-12-07T00:51:16,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676674 2024-12-07T00:51:16,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676674 2024-12-07T00:51:16,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676674) roll requested 2024-12-07T00:51:16,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676683 2024-12-07T00:51:16,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676683 2024-12-07T00:51:16,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676683) roll requested 2024-12-07T00:51:16,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676690 2024-12-07T00:51:16,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676690 2024-12-07T00:51:16,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676690) roll requested 2024-12-07T00:51:16,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676698 2024-12-07T00:51:16,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676698 2024-12-07T00:51:16,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676698) roll requested 2024-12-07T00:51:16,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676712 2024-12-07T00:51:16,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676712 2024-12-07T00:51:16,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676712) roll requested 2024-12-07T00:51:16,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676720 2024-12-07T00:51:16,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676720 2024-12-07T00:51:16,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676720) roll requested 2024-12-07T00:51:16,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676727 2024-12-07T00:51:16,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676727 2024-12-07T00:51:16,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676727) roll requested 2024-12-07T00:51:16,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676736 2024-12-07T00:51:16,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676736 2024-12-07T00:51:16,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676736) roll requested 2024-12-07T00:51:16,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676744 2024-12-07T00:51:16,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676744 2024-12-07T00:51:16,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676744) roll requested 2024-12-07T00:51:16,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676753 2024-12-07T00:51:16,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676753 2024-12-07T00:51:16,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676753) roll requested 2024-12-07T00:51:16,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676764 2024-12-07T00:51:16,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676764 2024-12-07T00:51:16,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676764) roll requested 2024-12-07T00:51:16,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676772 2024-12-07T00:51:16,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676772 2024-12-07T00:51:16,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676772) roll requested 2024-12-07T00:51:16,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676784 2024-12-07T00:51:16,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676784 2024-12-07T00:51:16,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676784) roll requested 2024-12-07T00:51:16,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676799 2024-12-07T00:51:16,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676799 2024-12-07T00:51:16,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676799) roll requested 2024-12-07T00:51:16,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676806 2024-12-07T00:51:16,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676806 2024-12-07T00:51:16,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676806) roll requested 2024-12-07T00:51:16,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676815 2024-12-07T00:51:16,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676815 2024-12-07T00:51:16,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676815) roll requested 2024-12-07T00:51:16,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676822 2024-12-07T00:51:16,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676822 2024-12-07T00:51:16,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676822) roll requested 2024-12-07T00:51:16,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676831 2024-12-07T00:51:16,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676831 2024-12-07T00:51:16,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676831) roll requested 2024-12-07T00:51:16,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676839 2024-12-07T00:51:16,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676839 2024-12-07T00:51:16,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676839) roll requested 2024-12-07T00:51:16,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676849 2024-12-07T00:51:16,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676849 2024-12-07T00:51:16,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676849) roll requested 2024-12-07T00:51:16,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676860 2024-12-07T00:51:16,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676860 2024-12-07T00:51:16,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676860) roll requested 2024-12-07T00:51:16,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676868 2024-12-07T00:51:16,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676868 2024-12-07T00:51:16,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676868) roll requested 2024-12-07T00:51:16,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676878 2024-12-07T00:51:16,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676878 2024-12-07T00:51:16,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676878) roll requested 2024-12-07T00:51:16,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676884 2024-12-07T00:51:16,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676884 2024-12-07T00:51:16,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676884) roll requested 2024-12-07T00:51:16,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676891 2024-12-07T00:51:16,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676891 2024-12-07T00:51:16,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676891) roll requested 2024-12-07T00:51:16,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676899 2024-12-07T00:51:16,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676899 2024-12-07T00:51:16,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676899) roll requested 2024-12-07T00:51:16,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676909 2024-12-07T00:51:16,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676909 2024-12-07T00:51:16,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676909) roll requested 2024-12-07T00:51:16,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676918 2024-12-07T00:51:16,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676918 2024-12-07T00:51:16,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676918) roll requested 2024-12-07T00:51:16,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676930 2024-12-07T00:51:16,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676930 2024-12-07T00:51:16,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676930) roll requested 2024-12-07T00:51:16,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676940 2024-12-07T00:51:16,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676940 2024-12-07T00:51:16,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676940) roll requested 2024-12-07T00:51:16,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676947 2024-12-07T00:51:16,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676947 2024-12-07T00:51:16,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676947) roll requested 2024-12-07T00:51:16,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676955 2024-12-07T00:51:16,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676955 2024-12-07T00:51:16,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676955) roll requested 2024-12-07T00:51:16,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676964 2024-12-07T00:51:16,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676964 2024-12-07T00:51:16,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676964) roll requested 2024-12-07T00:51:16,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676972 2024-12-07T00:51:16,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676972 2024-12-07T00:51:16,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676972) roll requested 2024-12-07T00:51:16,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676983 2024-12-07T00:51:16,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:16,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676983 2024-12-07T00:51:16,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:16,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:16,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676983) roll requested 2024-12-07T00:51:16,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532676992 2024-12-07T00:51:17,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676992 2024-12-07T00:51:17,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532676992) roll requested 2024-12-07T00:51:17,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677002 2024-12-07T00:51:17,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532676992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677002 2024-12-07T00:51:17,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677002) roll requested 2024-12-07T00:51:17,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677015 2024-12-07T00:51:17,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677015 2024-12-07T00:51:17,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677015) roll requested 2024-12-07T00:51:17,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677024 2024-12-07T00:51:17,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677024 2024-12-07T00:51:17,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677024) roll requested 2024-12-07T00:51:17,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677033 2024-12-07T00:51:17,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677033 2024-12-07T00:51:17,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677033) roll requested 2024-12-07T00:51:17,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677042 2024-12-07T00:51:17,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677042 2024-12-07T00:51:17,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677042) roll requested 2024-12-07T00:51:17,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677052 2024-12-07T00:51:17,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677052 2024-12-07T00:51:17,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677052) roll requested 2024-12-07T00:51:17,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677061 2024-12-07T00:51:17,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677061 2024-12-07T00:51:17,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677061) roll requested 2024-12-07T00:51:17,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677070 2024-12-07T00:51:17,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677070 2024-12-07T00:51:17,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677070) roll requested 2024-12-07T00:51:17,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677079 2024-12-07T00:51:17,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677079 2024-12-07T00:51:17,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677079) roll requested 2024-12-07T00:51:17,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677090 2024-12-07T00:51:17,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677090 2024-12-07T00:51:17,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677090) roll requested 2024-12-07T00:51:17,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677096 2024-12-07T00:51:17,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677096 2024-12-07T00:51:17,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677096) roll requested 2024-12-07T00:51:17,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677103 2024-12-07T00:51:17,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677103 2024-12-07T00:51:17,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677103) roll requested 2024-12-07T00:51:17,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677110 2024-12-07T00:51:17,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677110 2024-12-07T00:51:17,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677110) roll requested 2024-12-07T00:51:17,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677120 2024-12-07T00:51:17,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677120 2024-12-07T00:51:17,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677120) roll requested 2024-12-07T00:51:17,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677129 2024-12-07T00:51:17,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677129 2024-12-07T00:51:17,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677129) roll requested 2024-12-07T00:51:17,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677139 2024-12-07T00:51:17,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677139 2024-12-07T00:51:17,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677139) roll requested 2024-12-07T00:51:17,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677150 2024-12-07T00:51:17,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677150 2024-12-07T00:51:17,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677150) roll requested 2024-12-07T00:51:17,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677159 2024-12-07T00:51:17,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677159 2024-12-07T00:51:17,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677159) roll requested 2024-12-07T00:51:17,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677166 2024-12-07T00:51:17,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677166 2024-12-07T00:51:17,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677166) roll requested 2024-12-07T00:51:17,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677176 2024-12-07T00:51:17,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677176 2024-12-07T00:51:17,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677176) roll requested 2024-12-07T00:51:17,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677185 2024-12-07T00:51:17,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677185 2024-12-07T00:51:17,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677185) roll requested 2024-12-07T00:51:17,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677195 2024-12-07T00:51:17,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677195 2024-12-07T00:51:17,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677195) roll requested 2024-12-07T00:51:17,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677205 2024-12-07T00:51:17,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677205 2024-12-07T00:51:17,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677205) roll requested 2024-12-07T00:51:17,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677215 2024-12-07T00:51:17,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677215 2024-12-07T00:51:17,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677215) roll requested 2024-12-07T00:51:17,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677226 2024-12-07T00:51:17,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677226 2024-12-07T00:51:17,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677226) roll requested 2024-12-07T00:51:17,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677233 2024-12-07T00:51:17,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677233 2024-12-07T00:51:17,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677233) roll requested 2024-12-07T00:51:17,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677242 2024-12-07T00:51:17,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677242 2024-12-07T00:51:17,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677242) roll requested 2024-12-07T00:51:17,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677251 2024-12-07T00:51:17,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677251 2024-12-07T00:51:17,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677251) roll requested 2024-12-07T00:51:17,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677261 2024-12-07T00:51:17,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677261 2024-12-07T00:51:17,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677261) roll requested 2024-12-07T00:51:17,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677271 2024-12-07T00:51:17,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677271 2024-12-07T00:51:17,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677271) roll requested 2024-12-07T00:51:17,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677281 2024-12-07T00:51:17,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677281 2024-12-07T00:51:17,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677281) roll requested 2024-12-07T00:51:17,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677291 2024-12-07T00:51:17,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677291 2024-12-07T00:51:17,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677291) roll requested 2024-12-07T00:51:17,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677298 2024-12-07T00:51:17,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677298 2024-12-07T00:51:17,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677298) roll requested 2024-12-07T00:51:17,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677306 2024-12-07T00:51:17,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677306 2024-12-07T00:51:17,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677306) roll requested 2024-12-07T00:51:17,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677315 2024-12-07T00:51:17,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677315 2024-12-07T00:51:17,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677315) roll requested 2024-12-07T00:51:17,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677324 2024-12-07T00:51:17,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677324 2024-12-07T00:51:17,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677324) roll requested 2024-12-07T00:51:17,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677335 2024-12-07T00:51:17,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677335 2024-12-07T00:51:17,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677335) roll requested 2024-12-07T00:51:17,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677345 2024-12-07T00:51:17,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677345 2024-12-07T00:51:17,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677345) roll requested 2024-12-07T00:51:17,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677358 2024-12-07T00:51:17,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677358 2024-12-07T00:51:17,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677358) roll requested 2024-12-07T00:51:17,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677366 2024-12-07T00:51:17,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677366 2024-12-07T00:51:17,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677366) roll requested 2024-12-07T00:51:17,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677374 2024-12-07T00:51:17,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677374 2024-12-07T00:51:17,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677374) roll requested 2024-12-07T00:51:17,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677385 2024-12-07T00:51:17,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677385 2024-12-07T00:51:17,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677385) roll requested 2024-12-07T00:51:17,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677397 2024-12-07T00:51:17,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677397 2024-12-07T00:51:17,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677397) roll requested 2024-12-07T00:51:17,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677406 2024-12-07T00:51:17,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677406 2024-12-07T00:51:17,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677406) roll requested 2024-12-07T00:51:17,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677414 2024-12-07T00:51:17,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677414 2024-12-07T00:51:17,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677414) roll requested 2024-12-07T00:51:17,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677423 2024-12-07T00:51:17,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677423 2024-12-07T00:51:17,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677423) roll requested 2024-12-07T00:51:17,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677432 2024-12-07T00:51:17,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677432 2024-12-07T00:51:17,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677432) roll requested 2024-12-07T00:51:17,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677442 2024-12-07T00:51:17,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677442 2024-12-07T00:51:17,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677442) roll requested 2024-12-07T00:51:17,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677452 2024-12-07T00:51:17,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677452 2024-12-07T00:51:17,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677452) roll requested 2024-12-07T00:51:17,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677461 2024-12-07T00:51:17,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677461 2024-12-07T00:51:17,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677461) roll requested 2024-12-07T00:51:17,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677471 2024-12-07T00:51:17,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677471 2024-12-07T00:51:17,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677471) roll requested 2024-12-07T00:51:17,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677479 2024-12-07T00:51:17,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677479 2024-12-07T00:51:17,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677479) roll requested 2024-12-07T00:51:17,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677486 2024-12-07T00:51:17,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677486 2024-12-07T00:51:17,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677486) roll requested 2024-12-07T00:51:17,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677494 2024-12-07T00:51:17,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677494 2024-12-07T00:51:17,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677494) roll requested 2024-12-07T00:51:17,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677502 2024-12-07T00:51:17,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677502 2024-12-07T00:51:17,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677502) roll requested 2024-12-07T00:51:17,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677514 2024-12-07T00:51:17,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677514 2024-12-07T00:51:17,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677514) roll requested 2024-12-07T00:51:17,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677521 2024-12-07T00:51:17,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677521 2024-12-07T00:51:17,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677521) roll requested 2024-12-07T00:51:17,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677527 2024-12-07T00:51:17,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677527 2024-12-07T00:51:17,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677527) roll requested 2024-12-07T00:51:17,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677534 2024-12-07T00:51:17,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677534 2024-12-07T00:51:17,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677534) roll requested 2024-12-07T00:51:17,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677540 2024-12-07T00:51:17,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677540 2024-12-07T00:51:17,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677540) roll requested 2024-12-07T00:51:17,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677549 2024-12-07T00:51:17,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677549 2024-12-07T00:51:17,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677549) roll requested 2024-12-07T00:51:17,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677556 2024-12-07T00:51:17,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677556 2024-12-07T00:51:17,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677556) roll requested 2024-12-07T00:51:17,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677563 2024-12-07T00:51:17,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677563 2024-12-07T00:51:17,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677563) roll requested 2024-12-07T00:51:17,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677570 2024-12-07T00:51:17,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677570 2024-12-07T00:51:17,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677570) roll requested 2024-12-07T00:51:17,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677578 2024-12-07T00:51:17,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677578 2024-12-07T00:51:17,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677578) roll requested 2024-12-07T00:51:17,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677587 2024-12-07T00:51:17,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677587 2024-12-07T00:51:17,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677587) roll requested 2024-12-07T00:51:17,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677596 2024-12-07T00:51:17,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677596 2024-12-07T00:51:17,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677596) roll requested 2024-12-07T00:51:17,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677608 2024-12-07T00:51:17,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677608 2024-12-07T00:51:17,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677608) roll requested 2024-12-07T00:51:17,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677616 2024-12-07T00:51:17,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677616 2024-12-07T00:51:17,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677616) roll requested 2024-12-07T00:51:17,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677627 2024-12-07T00:51:17,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677627 2024-12-07T00:51:17,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677627) roll requested 2024-12-07T00:51:17,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677637 2024-12-07T00:51:17,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677637 2024-12-07T00:51:17,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677637) roll requested 2024-12-07T00:51:17,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677646 2024-12-07T00:51:17,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677646 2024-12-07T00:51:17,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677646) roll requested 2024-12-07T00:51:17,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677652 2024-12-07T00:51:17,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677652 2024-12-07T00:51:17,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677652) roll requested 2024-12-07T00:51:17,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677659 2024-12-07T00:51:17,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677659 2024-12-07T00:51:17,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677659) roll requested 2024-12-07T00:51:17,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677666 2024-12-07T00:51:17,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677666 2024-12-07T00:51:17,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677666) roll requested 2024-12-07T00:51:17,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677676 2024-12-07T00:51:17,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677676 2024-12-07T00:51:17,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677676) roll requested 2024-12-07T00:51:17,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677685 2024-12-07T00:51:17,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677685 2024-12-07T00:51:17,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677685) roll requested 2024-12-07T00:51:17,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677693 2024-12-07T00:51:17,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677693 2024-12-07T00:51:17,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677693) roll requested 2024-12-07T00:51:17,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677700 2024-12-07T00:51:17,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677700 2024-12-07T00:51:17,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677700) roll requested 2024-12-07T00:51:17,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677706 2024-12-07T00:51:17,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677706 2024-12-07T00:51:17,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677706) roll requested 2024-12-07T00:51:17,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677714 2024-12-07T00:51:17,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677714 2024-12-07T00:51:17,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677714) roll requested 2024-12-07T00:51:17,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677725 2024-12-07T00:51:17,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677725 2024-12-07T00:51:17,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677725) roll requested 2024-12-07T00:51:17,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677732 2024-12-07T00:51:17,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677732 2024-12-07T00:51:17,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677732) roll requested 2024-12-07T00:51:17,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677739 2024-12-07T00:51:17,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677739 2024-12-07T00:51:17,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677739) roll requested 2024-12-07T00:51:17,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677746 2024-12-07T00:51:17,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677746 2024-12-07T00:51:17,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677746) roll requested 2024-12-07T00:51:17,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677752 2024-12-07T00:51:17,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677752 2024-12-07T00:51:17,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677752) roll requested 2024-12-07T00:51:17,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677763 2024-12-07T00:51:17,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677763 2024-12-07T00:51:17,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677763) roll requested 2024-12-07T00:51:17,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677772 2024-12-07T00:51:17,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677772 2024-12-07T00:51:17,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677772) roll requested 2024-12-07T00:51:17,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677784 2024-12-07T00:51:17,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677784 2024-12-07T00:51:17,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677784) roll requested 2024-12-07T00:51:17,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677797 2024-12-07T00:51:17,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677797 2024-12-07T00:51:17,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677797) roll requested 2024-12-07T00:51:17,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677804 2024-12-07T00:51:17,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677804 2024-12-07T00:51:17,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677804) roll requested 2024-12-07T00:51:17,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677813 2024-12-07T00:51:17,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677813 2024-12-07T00:51:17,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677813) roll requested 2024-12-07T00:51:17,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677820 2024-12-07T00:51:17,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677820 2024-12-07T00:51:17,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677820) roll requested 2024-12-07T00:51:17,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677829 2024-12-07T00:51:17,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677829 2024-12-07T00:51:17,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677829) roll requested 2024-12-07T00:51:17,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677837 2024-12-07T00:51:17,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677837 2024-12-07T00:51:17,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677837) roll requested 2024-12-07T00:51:17,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677848 2024-12-07T00:51:17,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677848 2024-12-07T00:51:17,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677848) roll requested 2024-12-07T00:51:17,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677857 2024-12-07T00:51:17,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677857 2024-12-07T00:51:17,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677857) roll requested 2024-12-07T00:51:17,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677865 2024-12-07T00:51:17,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677865 2024-12-07T00:51:17,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677865) roll requested 2024-12-07T00:51:17,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677874 2024-12-07T00:51:17,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677874 2024-12-07T00:51:17,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677874) roll requested 2024-12-07T00:51:17,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677881 2024-12-07T00:51:17,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677881 2024-12-07T00:51:17,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677881) roll requested 2024-12-07T00:51:17,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677889 2024-12-07T00:51:17,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677889 2024-12-07T00:51:17,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677889) roll requested 2024-12-07T00:51:17,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677895 2024-12-07T00:51:17,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677895 2024-12-07T00:51:17,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677895) roll requested 2024-12-07T00:51:17,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677905 2024-12-07T00:51:17,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677905 2024-12-07T00:51:17,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677905) roll requested 2024-12-07T00:51:17,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677917 2024-12-07T00:51:17,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677917 2024-12-07T00:51:17,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677917) roll requested 2024-12-07T00:51:17,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677928 2024-12-07T00:51:17,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677928 2024-12-07T00:51:17,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677928) roll requested 2024-12-07T00:51:17,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677936 2024-12-07T00:51:17,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677936 2024-12-07T00:51:17,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677936) roll requested 2024-12-07T00:51:17,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677945 2024-12-07T00:51:17,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677945 2024-12-07T00:51:17,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677945) roll requested 2024-12-07T00:51:17,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677952 2024-12-07T00:51:17,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677952 2024-12-07T00:51:17,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677952) roll requested 2024-12-07T00:51:17,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677959 2024-12-07T00:51:17,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677959 2024-12-07T00:51:17,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677959) roll requested 2024-12-07T00:51:17,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677969 2024-12-07T00:51:17,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677969 2024-12-07T00:51:17,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677969) roll requested 2024-12-07T00:51:17,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677976 2024-12-07T00:51:17,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677976 2024-12-07T00:51:17,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677976) roll requested 2024-12-07T00:51:17,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677983 2024-12-07T00:51:17,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677983 2024-12-07T00:51:17,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:17,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:17,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677983) roll requested 2024-12-07T00:51:17,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532677990 2024-12-07T00:51:17,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:17,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677990 2024-12-07T00:51:18,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532677990) roll requested 2024-12-07T00:51:18,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678001 2024-12-07T00:51:18,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532677990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678001 2024-12-07T00:51:18,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678001) roll requested 2024-12-07T00:51:18,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678008 2024-12-07T00:51:18,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678008 2024-12-07T00:51:18,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678008) roll requested 2024-12-07T00:51:18,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678016 2024-12-07T00:51:18,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678016 2024-12-07T00:51:18,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678016) roll requested 2024-12-07T00:51:18,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678023 2024-12-07T00:51:18,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678023 2024-12-07T00:51:18,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678023) roll requested 2024-12-07T00:51:18,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678030 2024-12-07T00:51:18,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678030 2024-12-07T00:51:18,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678030) roll requested 2024-12-07T00:51:18,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678037 2024-12-07T00:51:18,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678037 2024-12-07T00:51:18,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678037) roll requested 2024-12-07T00:51:18,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678044 2024-12-07T00:51:18,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678044 2024-12-07T00:51:18,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678044) roll requested 2024-12-07T00:51:18,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678053 2024-12-07T00:51:18,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678053 2024-12-07T00:51:18,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678053) roll requested 2024-12-07T00:51:18,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678059 2024-12-07T00:51:18,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678059 2024-12-07T00:51:18,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678059) roll requested 2024-12-07T00:51:18,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678066 2024-12-07T00:51:18,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678066 2024-12-07T00:51:18,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678066) roll requested 2024-12-07T00:51:18,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678072 2024-12-07T00:51:18,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678072 2024-12-07T00:51:18,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678072) roll requested 2024-12-07T00:51:18,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678079 2024-12-07T00:51:18,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678079 2024-12-07T00:51:18,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678079) roll requested 2024-12-07T00:51:18,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678085 2024-12-07T00:51:18,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678085 2024-12-07T00:51:18,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678085) roll requested 2024-12-07T00:51:18,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678093 2024-12-07T00:51:18,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678093 2024-12-07T00:51:18,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678093) roll requested 2024-12-07T00:51:18,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678099 2024-12-07T00:51:18,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678099 2024-12-07T00:51:18,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678099) roll requested 2024-12-07T00:51:18,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678106 2024-12-07T00:51:18,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678106 2024-12-07T00:51:18,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678106) roll requested 2024-12-07T00:51:18,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678115 2024-12-07T00:51:18,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678115 2024-12-07T00:51:18,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678115) roll requested 2024-12-07T00:51:18,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678122 2024-12-07T00:51:18,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678122 2024-12-07T00:51:18,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678122) roll requested 2024-12-07T00:51:18,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678130 2024-12-07T00:51:18,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678130 2024-12-07T00:51:18,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678130) roll requested 2024-12-07T00:51:18,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678139 2024-12-07T00:51:18,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678139 2024-12-07T00:51:18,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678139) roll requested 2024-12-07T00:51:18,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678153 2024-12-07T00:51:18,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678153 2024-12-07T00:51:18,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678153) roll requested 2024-12-07T00:51:18,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678160 2024-12-07T00:51:18,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678160 2024-12-07T00:51:18,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678160) roll requested 2024-12-07T00:51:18,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678167 2024-12-07T00:51:18,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678167 2024-12-07T00:51:18,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678167) roll requested 2024-12-07T00:51:18,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678175 2024-12-07T00:51:18,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678175 2024-12-07T00:51:18,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678175) roll requested 2024-12-07T00:51:18,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678183 2024-12-07T00:51:18,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678183 2024-12-07T00:51:18,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678183) roll requested 2024-12-07T00:51:18,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678193 2024-12-07T00:51:18,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678193 2024-12-07T00:51:18,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678193) roll requested 2024-12-07T00:51:18,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678202 2024-12-07T00:51:18,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678202 2024-12-07T00:51:18,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678202) roll requested 2024-12-07T00:51:18,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678214 2024-12-07T00:51:18,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678214 2024-12-07T00:51:18,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678214) roll requested 2024-12-07T00:51:18,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678226 2024-12-07T00:51:18,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678226 2024-12-07T00:51:18,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678226) roll requested 2024-12-07T00:51:18,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678237 2024-12-07T00:51:18,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678237 2024-12-07T00:51:18,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678237) roll requested 2024-12-07T00:51:18,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678247 2024-12-07T00:51:18,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678247 2024-12-07T00:51:18,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678247) roll requested 2024-12-07T00:51:18,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678254 2024-12-07T00:51:18,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678254 2024-12-07T00:51:18,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678254) roll requested 2024-12-07T00:51:18,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678262 2024-12-07T00:51:18,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678262 2024-12-07T00:51:18,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678262) roll requested 2024-12-07T00:51:18,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678271 2024-12-07T00:51:18,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678271 2024-12-07T00:51:18,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678271) roll requested 2024-12-07T00:51:18,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678279 2024-12-07T00:51:18,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678279 2024-12-07T00:51:18,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678279) roll requested 2024-12-07T00:51:18,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678286 2024-12-07T00:51:18,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678286 2024-12-07T00:51:18,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678286) roll requested 2024-12-07T00:51:18,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678295 2024-12-07T00:51:18,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678295 2024-12-07T00:51:18,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678295) roll requested 2024-12-07T00:51:18,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678304 2024-12-07T00:51:18,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678304 2024-12-07T00:51:18,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678304) roll requested 2024-12-07T00:51:18,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678313 2024-12-07T00:51:18,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678313 2024-12-07T00:51:18,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678313) roll requested 2024-12-07T00:51:18,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678321 2024-12-07T00:51:18,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678321 2024-12-07T00:51:18,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678321) roll requested 2024-12-07T00:51:18,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678332 2024-12-07T00:51:18,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678332 2024-12-07T00:51:18,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678332) roll requested 2024-12-07T00:51:18,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678343 2024-12-07T00:51:18,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678343 2024-12-07T00:51:18,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678343) roll requested 2024-12-07T00:51:18,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678352 2024-12-07T00:51:18,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678352 2024-12-07T00:51:18,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678352) roll requested 2024-12-07T00:51:18,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678362 2024-12-07T00:51:18,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678362 2024-12-07T00:51:18,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678362) roll requested 2024-12-07T00:51:18,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678371 2024-12-07T00:51:18,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678371 2024-12-07T00:51:18,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678371) roll requested 2024-12-07T00:51:18,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678378 2024-12-07T00:51:18,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678378 2024-12-07T00:51:18,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678378) roll requested 2024-12-07T00:51:18,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678384 2024-12-07T00:51:18,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678384 2024-12-07T00:51:18,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678384) roll requested 2024-12-07T00:51:18,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678394 2024-12-07T00:51:18,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678394 2024-12-07T00:51:18,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678394) roll requested 2024-12-07T00:51:18,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678402 2024-12-07T00:51:18,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678402 2024-12-07T00:51:18,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678402) roll requested 2024-12-07T00:51:18,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678413 2024-12-07T00:51:18,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678413 2024-12-07T00:51:18,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678413) roll requested 2024-12-07T00:51:18,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678421 2024-12-07T00:51:18,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678421 2024-12-07T00:51:18,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678421) roll requested 2024-12-07T00:51:18,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678427 2024-12-07T00:51:18,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678427 2024-12-07T00:51:18,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678427) roll requested 2024-12-07T00:51:18,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678434 2024-12-07T00:51:18,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678434 2024-12-07T00:51:18,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678434) roll requested 2024-12-07T00:51:18,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678441 2024-12-07T00:51:18,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678441 2024-12-07T00:51:18,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678441) roll requested 2024-12-07T00:51:18,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678449 2024-12-07T00:51:18,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678449 2024-12-07T00:51:18,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678449) roll requested 2024-12-07T00:51:18,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678465 2024-12-07T00:51:18,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678465 2024-12-07T00:51:18,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678465) roll requested 2024-12-07T00:51:18,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678472 2024-12-07T00:51:18,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678472 2024-12-07T00:51:18,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678472) roll requested 2024-12-07T00:51:18,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678479 2024-12-07T00:51:18,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678479 2024-12-07T00:51:18,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678479) roll requested 2024-12-07T00:51:18,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678486 2024-12-07T00:51:18,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678486 2024-12-07T00:51:18,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678486) roll requested 2024-12-07T00:51:18,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678495 2024-12-07T00:51:18,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678495 2024-12-07T00:51:18,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678495) roll requested 2024-12-07T00:51:18,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678505 2024-12-07T00:51:18,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678505 2024-12-07T00:51:18,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678505) roll requested 2024-12-07T00:51:18,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678512 2024-12-07T00:51:18,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678512 2024-12-07T00:51:18,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678512) roll requested 2024-12-07T00:51:18,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678525 2024-12-07T00:51:18,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678525 2024-12-07T00:51:18,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678525) roll requested 2024-12-07T00:51:18,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678532 2024-12-07T00:51:18,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678532 2024-12-07T00:51:18,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678532) roll requested 2024-12-07T00:51:18,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678541 2024-12-07T00:51:18,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678541 2024-12-07T00:51:18,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678541) roll requested 2024-12-07T00:51:18,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678550 2024-12-07T00:51:18,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678550 2024-12-07T00:51:18,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678550) roll requested 2024-12-07T00:51:18,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678558 2024-12-07T00:51:18,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678558 2024-12-07T00:51:18,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678558) roll requested 2024-12-07T00:51:18,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678568 2024-12-07T00:51:18,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678568 2024-12-07T00:51:18,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678568) roll requested 2024-12-07T00:51:18,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678575 2024-12-07T00:51:18,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678575 2024-12-07T00:51:18,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678575) roll requested 2024-12-07T00:51:18,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678581 2024-12-07T00:51:18,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678581 2024-12-07T00:51:18,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678581) roll requested 2024-12-07T00:51:18,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678588 2024-12-07T00:51:18,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678588 2024-12-07T00:51:18,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678588) roll requested 2024-12-07T00:51:18,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678595 2024-12-07T00:51:18,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678595 2024-12-07T00:51:18,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678595) roll requested 2024-12-07T00:51:18,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678603 2024-12-07T00:51:18,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678603 2024-12-07T00:51:18,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678603) roll requested 2024-12-07T00:51:18,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678612 2024-12-07T00:51:18,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678612 2024-12-07T00:51:18,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678612) roll requested 2024-12-07T00:51:18,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678620 2024-12-07T00:51:18,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678620 2024-12-07T00:51:18,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678620) roll requested 2024-12-07T00:51:18,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678629 2024-12-07T00:51:18,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678629 2024-12-07T00:51:18,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678629) roll requested 2024-12-07T00:51:18,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678636 2024-12-07T00:51:18,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678636 2024-12-07T00:51:18,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678636) roll requested 2024-12-07T00:51:18,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678645 2024-12-07T00:51:18,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678645 2024-12-07T00:51:18,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678645) roll requested 2024-12-07T00:51:18,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678652 2024-12-07T00:51:18,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678652 2024-12-07T00:51:18,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678652) roll requested 2024-12-07T00:51:18,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678660 2024-12-07T00:51:18,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678660 2024-12-07T00:51:18,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678660) roll requested 2024-12-07T00:51:18,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678669 2024-12-07T00:51:18,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678669 2024-12-07T00:51:18,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678669) roll requested 2024-12-07T00:51:18,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678679 2024-12-07T00:51:18,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678679 2024-12-07T00:51:18,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678679) roll requested 2024-12-07T00:51:18,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678690 2024-12-07T00:51:18,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678690 2024-12-07T00:51:18,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678690) roll requested 2024-12-07T00:51:18,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678701 2024-12-07T00:51:18,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678701 2024-12-07T00:51:18,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678701) roll requested 2024-12-07T00:51:18,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678714 2024-12-07T00:51:18,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678714 2024-12-07T00:51:18,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678714) roll requested 2024-12-07T00:51:18,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678720 2024-12-07T00:51:18,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678720 2024-12-07T00:51:18,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678720) roll requested 2024-12-07T00:51:18,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678726 2024-12-07T00:51:18,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678726 2024-12-07T00:51:18,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678726) roll requested 2024-12-07T00:51:18,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678735 2024-12-07T00:51:18,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678735 2024-12-07T00:51:18,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678735) roll requested 2024-12-07T00:51:18,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678746 2024-12-07T00:51:18,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678746 2024-12-07T00:51:18,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678746) roll requested 2024-12-07T00:51:18,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678755 2024-12-07T00:51:18,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678755 2024-12-07T00:51:18,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678755) roll requested 2024-12-07T00:51:18,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678765 2024-12-07T00:51:18,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678765 2024-12-07T00:51:18,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678765) roll requested 2024-12-07T00:51:18,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678772 2024-12-07T00:51:18,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678772 2024-12-07T00:51:18,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678772) roll requested 2024-12-07T00:51:18,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678779 2024-12-07T00:51:18,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678779 2024-12-07T00:51:18,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678779) roll requested 2024-12-07T00:51:18,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678785 2024-12-07T00:51:18,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678785 2024-12-07T00:51:18,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678785) roll requested 2024-12-07T00:51:18,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678791 2024-12-07T00:51:18,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678791 2024-12-07T00:51:18,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678791) roll requested 2024-12-07T00:51:18,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678798 2024-12-07T00:51:18,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678798 2024-12-07T00:51:18,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678798) roll requested 2024-12-07T00:51:18,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678808 2024-12-07T00:51:18,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678808 2024-12-07T00:51:18,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678808) roll requested 2024-12-07T00:51:18,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678816 2024-12-07T00:51:18,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678816 2024-12-07T00:51:18,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678816) roll requested 2024-12-07T00:51:18,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678828 2024-12-07T00:51:18,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678828 2024-12-07T00:51:18,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678828) roll requested 2024-12-07T00:51:18,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678835 2024-12-07T00:51:18,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678835 2024-12-07T00:51:18,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678835) roll requested 2024-12-07T00:51:18,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678842 2024-12-07T00:51:18,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678842 2024-12-07T00:51:18,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678842) roll requested 2024-12-07T00:51:18,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678849 2024-12-07T00:51:18,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678849 2024-12-07T00:51:18,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678849) roll requested 2024-12-07T00:51:18,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678858 2024-12-07T00:51:18,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678858 2024-12-07T00:51:18,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678858) roll requested 2024-12-07T00:51:18,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678865 2024-12-07T00:51:18,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678865 2024-12-07T00:51:18,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678865) roll requested 2024-12-07T00:51:18,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678872 2024-12-07T00:51:18,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678872 2024-12-07T00:51:18,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678872) roll requested 2024-12-07T00:51:18,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678878 2024-12-07T00:51:18,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678878 2024-12-07T00:51:18,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678878) roll requested 2024-12-07T00:51:18,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678884 2024-12-07T00:51:18,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678884 2024-12-07T00:51:18,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678884) roll requested 2024-12-07T00:51:18,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678891 2024-12-07T00:51:18,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678891 2024-12-07T00:51:18,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678891) roll requested 2024-12-07T00:51:18,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678899 2024-12-07T00:51:18,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678899 2024-12-07T00:51:18,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678899) roll requested 2024-12-07T00:51:18,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678907 2024-12-07T00:51:18,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678907 2024-12-07T00:51:18,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678907) roll requested 2024-12-07T00:51:18,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678916 2024-12-07T00:51:18,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678916 2024-12-07T00:51:18,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678916) roll requested 2024-12-07T00:51:18,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678927 2024-12-07T00:51:18,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678927 2024-12-07T00:51:18,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678927) roll requested 2024-12-07T00:51:18,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678936 2024-12-07T00:51:18,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678936 2024-12-07T00:51:18,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678936) roll requested 2024-12-07T00:51:18,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678948 2024-12-07T00:51:18,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678948 2024-12-07T00:51:18,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678948) roll requested 2024-12-07T00:51:18,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678958 2024-12-07T00:51:18,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678958 2024-12-07T00:51:18,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678958) roll requested 2024-12-07T00:51:18,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678966 2024-12-07T00:51:18,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678966 2024-12-07T00:51:18,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678966) roll requested 2024-12-07T00:51:18,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678976 2024-12-07T00:51:18,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678976 2024-12-07T00:51:18,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678976) roll requested 2024-12-07T00:51:18,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678981 2024-12-07T00:51:18,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678981 2024-12-07T00:51:18,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:18,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678981) roll requested 2024-12-07T00:51:18,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532678990 2024-12-07T00:51:18,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:18,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678990 2024-12-07T00:51:18,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:18,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532678990) roll requested 2024-12-07T00:51:19,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679000 2024-12-07T00:51:19,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532678990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679000 2024-12-07T00:51:19,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679000) roll requested 2024-12-07T00:51:19,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679010 2024-12-07T00:51:19,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679010 2024-12-07T00:51:19,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679010) roll requested 2024-12-07T00:51:19,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679017 2024-12-07T00:51:19,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679017 2024-12-07T00:51:19,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679017) roll requested 2024-12-07T00:51:19,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679023 2024-12-07T00:51:19,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679023 2024-12-07T00:51:19,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679023) roll requested 2024-12-07T00:51:19,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679032 2024-12-07T00:51:19,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679032 2024-12-07T00:51:19,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679032) roll requested 2024-12-07T00:51:19,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679039 2024-12-07T00:51:19,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679039 2024-12-07T00:51:19,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679039) roll requested 2024-12-07T00:51:19,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679047 2024-12-07T00:51:19,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679047 2024-12-07T00:51:19,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679047) roll requested 2024-12-07T00:51:19,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679054 2024-12-07T00:51:19,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679054 2024-12-07T00:51:19,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679054) roll requested 2024-12-07T00:51:19,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679059 2024-12-07T00:51:19,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679059 2024-12-07T00:51:19,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679059) roll requested 2024-12-07T00:51:19,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679065 2024-12-07T00:51:19,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679065 2024-12-07T00:51:19,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679065) roll requested 2024-12-07T00:51:19,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679072 2024-12-07T00:51:19,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679072 2024-12-07T00:51:19,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679072) roll requested 2024-12-07T00:51:19,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679078 2024-12-07T00:51:19,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679078 2024-12-07T00:51:19,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679078) roll requested 2024-12-07T00:51:19,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679087 2024-12-07T00:51:19,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679087 2024-12-07T00:51:19,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679087) roll requested 2024-12-07T00:51:19,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679094 2024-12-07T00:51:19,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679094 2024-12-07T00:51:19,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679094) roll requested 2024-12-07T00:51:19,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679101 2024-12-07T00:51:19,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679101 2024-12-07T00:51:19,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679101) roll requested 2024-12-07T00:51:19,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679109 2024-12-07T00:51:19,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679109 2024-12-07T00:51:19,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679109) roll requested 2024-12-07T00:51:19,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679117 2024-12-07T00:51:19,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679117 2024-12-07T00:51:19,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679117) roll requested 2024-12-07T00:51:19,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679128 2024-12-07T00:51:19,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679128 2024-12-07T00:51:19,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679128) roll requested 2024-12-07T00:51:19,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679139 2024-12-07T00:51:19,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679139 2024-12-07T00:51:19,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679139) roll requested 2024-12-07T00:51:19,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679149 2024-12-07T00:51:19,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679149 2024-12-07T00:51:19,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679149) roll requested 2024-12-07T00:51:19,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679159 2024-12-07T00:51:19,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679159 2024-12-07T00:51:19,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679159) roll requested 2024-12-07T00:51:19,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679169 2024-12-07T00:51:19,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679169 2024-12-07T00:51:19,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679169) roll requested 2024-12-07T00:51:19,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679180 2024-12-07T00:51:19,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679180 2024-12-07T00:51:19,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679180) roll requested 2024-12-07T00:51:19,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679192 2024-12-07T00:51:19,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679192 2024-12-07T00:51:19,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679192) roll requested 2024-12-07T00:51:19,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679203 2024-12-07T00:51:19,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679203 2024-12-07T00:51:19,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679203) roll requested 2024-12-07T00:51:19,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679211 2024-12-07T00:51:19,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679211 2024-12-07T00:51:19,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679211) roll requested 2024-12-07T00:51:19,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679219 2024-12-07T00:51:19,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679219 2024-12-07T00:51:19,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679219) roll requested 2024-12-07T00:51:19,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679226 2024-12-07T00:51:19,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679226 2024-12-07T00:51:19,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679226) roll requested 2024-12-07T00:51:19,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679236 2024-12-07T00:51:19,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679236 2024-12-07T00:51:19,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679236) roll requested 2024-12-07T00:51:19,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679243 2024-12-07T00:51:19,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679243 2024-12-07T00:51:19,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679243) roll requested 2024-12-07T00:51:19,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679250 2024-12-07T00:51:19,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679250 2024-12-07T00:51:19,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679250) roll requested 2024-12-07T00:51:19,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679259 2024-12-07T00:51:19,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679259 2024-12-07T00:51:19,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679259) roll requested 2024-12-07T00:51:19,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679267 2024-12-07T00:51:19,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679267 2024-12-07T00:51:19,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679267) roll requested 2024-12-07T00:51:19,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679276 2024-12-07T00:51:19,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679276 2024-12-07T00:51:19,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679276) roll requested 2024-12-07T00:51:19,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679285 2024-12-07T00:51:19,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679285 2024-12-07T00:51:19,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679285) roll requested 2024-12-07T00:51:19,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679295 2024-12-07T00:51:19,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679295 2024-12-07T00:51:19,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679295) roll requested 2024-12-07T00:51:19,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679304 2024-12-07T00:51:19,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679304 2024-12-07T00:51:19,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679304) roll requested 2024-12-07T00:51:19,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679310 2024-12-07T00:51:19,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679310 2024-12-07T00:51:19,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679310) roll requested 2024-12-07T00:51:19,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679316 2024-12-07T00:51:19,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679316 2024-12-07T00:51:19,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679316) roll requested 2024-12-07T00:51:19,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679323 2024-12-07T00:51:19,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679323 2024-12-07T00:51:19,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679323) roll requested 2024-12-07T00:51:19,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679331 2024-12-07T00:51:19,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679331 2024-12-07T00:51:19,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679331) roll requested 2024-12-07T00:51:19,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679338 2024-12-07T00:51:19,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679338 2024-12-07T00:51:19,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679338) roll requested 2024-12-07T00:51:19,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679345 2024-12-07T00:51:19,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679345 2024-12-07T00:51:19,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679345) roll requested 2024-12-07T00:51:19,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679351 2024-12-07T00:51:19,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679351 2024-12-07T00:51:19,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679351) roll requested 2024-12-07T00:51:19,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679359 2024-12-07T00:51:19,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679359 2024-12-07T00:51:19,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679359) roll requested 2024-12-07T00:51:19,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679368 2024-12-07T00:51:19,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679368 2024-12-07T00:51:19,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679368) roll requested 2024-12-07T00:51:19,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679374 2024-12-07T00:51:19,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679374 2024-12-07T00:51:19,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679374) roll requested 2024-12-07T00:51:19,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679380 2024-12-07T00:51:19,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679380 2024-12-07T00:51:19,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679380) roll requested 2024-12-07T00:51:19,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679386 2024-12-07T00:51:19,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679386 2024-12-07T00:51:19,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679386) roll requested 2024-12-07T00:51:19,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679394 2024-12-07T00:51:19,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679394 2024-12-07T00:51:19,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679394) roll requested 2024-12-07T00:51:19,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679400 2024-12-07T00:51:19,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679400 2024-12-07T00:51:19,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679400) roll requested 2024-12-07T00:51:19,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679408 2024-12-07T00:51:19,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679408 2024-12-07T00:51:19,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679408) roll requested 2024-12-07T00:51:19,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679413 2024-12-07T00:51:19,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679413 2024-12-07T00:51:19,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679413) roll requested 2024-12-07T00:51:19,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679419 2024-12-07T00:51:19,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679419 2024-12-07T00:51:19,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679419) roll requested 2024-12-07T00:51:19,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679426 2024-12-07T00:51:19,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679426 2024-12-07T00:51:19,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679426) roll requested 2024-12-07T00:51:19,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679433 2024-12-07T00:51:19,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679433 2024-12-07T00:51:19,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679433) roll requested 2024-12-07T00:51:19,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679442 2024-12-07T00:51:19,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679442 2024-12-07T00:51:19,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679442) roll requested 2024-12-07T00:51:19,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679449 2024-12-07T00:51:19,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679449 2024-12-07T00:51:19,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679449) roll requested 2024-12-07T00:51:19,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679455 2024-12-07T00:51:19,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679455 2024-12-07T00:51:19,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679455) roll requested 2024-12-07T00:51:19,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679460 2024-12-07T00:51:19,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679460 2024-12-07T00:51:19,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679460) roll requested 2024-12-07T00:51:19,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679466 2024-12-07T00:51:19,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679466 2024-12-07T00:51:19,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679466) roll requested 2024-12-07T00:51:19,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679474 2024-12-07T00:51:19,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679474 2024-12-07T00:51:19,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679474) roll requested 2024-12-07T00:51:19,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679482 2024-12-07T00:51:19,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679482 2024-12-07T00:51:19,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679482) roll requested 2024-12-07T00:51:19,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679491 2024-12-07T00:51:19,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679491 2024-12-07T00:51:19,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679491) roll requested 2024-12-07T00:51:19,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679499 2024-12-07T00:51:19,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679499 2024-12-07T00:51:19,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679499) roll requested 2024-12-07T00:51:19,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679509 2024-12-07T00:51:19,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679509 2024-12-07T00:51:19,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679509) roll requested 2024-12-07T00:51:19,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679518 2024-12-07T00:51:19,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679518 2024-12-07T00:51:19,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679518) roll requested 2024-12-07T00:51:19,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679527 2024-12-07T00:51:19,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679527 2024-12-07T00:51:19,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679527) roll requested 2024-12-07T00:51:19,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679536 2024-12-07T00:51:19,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679536 2024-12-07T00:51:19,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679536) roll requested 2024-12-07T00:51:19,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679543 2024-12-07T00:51:19,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679543 2024-12-07T00:51:19,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679543) roll requested 2024-12-07T00:51:19,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679557 2024-12-07T00:51:19,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679557 2024-12-07T00:51:19,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679557) roll requested 2024-12-07T00:51:19,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679572 2024-12-07T00:51:19,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679572 2024-12-07T00:51:19,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679572) roll requested 2024-12-07T00:51:19,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679579 2024-12-07T00:51:19,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679579 2024-12-07T00:51:19,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679579) roll requested 2024-12-07T00:51:19,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679588 2024-12-07T00:51:19,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679588 2024-12-07T00:51:19,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679588) roll requested 2024-12-07T00:51:19,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679597 2024-12-07T00:51:19,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679597 2024-12-07T00:51:19,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679597) roll requested 2024-12-07T00:51:19,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679604 2024-12-07T00:51:19,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679604 2024-12-07T00:51:19,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679604) roll requested 2024-12-07T00:51:19,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679611 2024-12-07T00:51:19,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679611 2024-12-07T00:51:19,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679611) roll requested 2024-12-07T00:51:19,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679623 2024-12-07T00:51:19,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679623 2024-12-07T00:51:19,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679623) roll requested 2024-12-07T00:51:19,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679631 2024-12-07T00:51:19,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679631 2024-12-07T00:51:19,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679631) roll requested 2024-12-07T00:51:19,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679636 2024-12-07T00:51:19,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679636 2024-12-07T00:51:19,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679636) roll requested 2024-12-07T00:51:19,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679642 2024-12-07T00:51:19,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679642 2024-12-07T00:51:19,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679642) roll requested 2024-12-07T00:51:19,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679647 2024-12-07T00:51:19,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679647 2024-12-07T00:51:19,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679647) roll requested 2024-12-07T00:51:19,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679654 2024-12-07T00:51:19,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679654 2024-12-07T00:51:19,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679654) roll requested 2024-12-07T00:51:19,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679661 2024-12-07T00:51:19,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679661 2024-12-07T00:51:19,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679661) roll requested 2024-12-07T00:51:19,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679667 2024-12-07T00:51:19,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679667 2024-12-07T00:51:19,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679667) roll requested 2024-12-07T00:51:19,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679674 2024-12-07T00:51:19,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679674 2024-12-07T00:51:19,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679674) roll requested 2024-12-07T00:51:19,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679683 2024-12-07T00:51:19,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679683 2024-12-07T00:51:19,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679683) roll requested 2024-12-07T00:51:19,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679693 2024-12-07T00:51:19,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679693 2024-12-07T00:51:19,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679693) roll requested 2024-12-07T00:51:19,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679704 2024-12-07T00:51:19,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679704 2024-12-07T00:51:19,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679704) roll requested 2024-12-07T00:51:19,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679712 2024-12-07T00:51:19,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679712 2024-12-07T00:51:19,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679712) roll requested 2024-12-07T00:51:19,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679720 2024-12-07T00:51:19,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679720 2024-12-07T00:51:19,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679720) roll requested 2024-12-07T00:51:19,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679728 2024-12-07T00:51:19,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679728 2024-12-07T00:51:19,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679728) roll requested 2024-12-07T00:51:19,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679736 2024-12-07T00:51:19,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679736 2024-12-07T00:51:19,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679736) roll requested 2024-12-07T00:51:19,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679745 2024-12-07T00:51:19,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679745 2024-12-07T00:51:19,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679745) roll requested 2024-12-07T00:51:19,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679755 2024-12-07T00:51:19,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679755 2024-12-07T00:51:19,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679755) roll requested 2024-12-07T00:51:19,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679762 2024-12-07T00:51:19,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679762 2024-12-07T00:51:19,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679762) roll requested 2024-12-07T00:51:19,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679772 2024-12-07T00:51:19,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679772 2024-12-07T00:51:19,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679772) roll requested 2024-12-07T00:51:19,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679780 2024-12-07T00:51:19,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679780 2024-12-07T00:51:19,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679780) roll requested 2024-12-07T00:51:19,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679788 2024-12-07T00:51:19,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679788 2024-12-07T00:51:19,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679788) roll requested 2024-12-07T00:51:19,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679793 2024-12-07T00:51:19,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679793 2024-12-07T00:51:19,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679793) roll requested 2024-12-07T00:51:19,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679801 2024-12-07T00:51:19,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679801 2024-12-07T00:51:19,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679801) roll requested 2024-12-07T00:51:19,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679809 2024-12-07T00:51:19,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679809 2024-12-07T00:51:19,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679809) roll requested 2024-12-07T00:51:19,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679817 2024-12-07T00:51:19,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679817 2024-12-07T00:51:19,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679817) roll requested 2024-12-07T00:51:19,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679827 2024-12-07T00:51:19,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679827 2024-12-07T00:51:19,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679827) roll requested 2024-12-07T00:51:19,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679838 2024-12-07T00:51:19,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679838 2024-12-07T00:51:19,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679838) roll requested 2024-12-07T00:51:19,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679845 2024-12-07T00:51:19,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679845 2024-12-07T00:51:19,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679845) roll requested 2024-12-07T00:51:19,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679851 2024-12-07T00:51:19,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679851 2024-12-07T00:51:19,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679851) roll requested 2024-12-07T00:51:19,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679857 2024-12-07T00:51:19,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679857 2024-12-07T00:51:19,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679857) roll requested 2024-12-07T00:51:19,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679865 2024-12-07T00:51:19,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679865 2024-12-07T00:51:19,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679865) roll requested 2024-12-07T00:51:19,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679874 2024-12-07T00:51:19,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679874 2024-12-07T00:51:19,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679874) roll requested 2024-12-07T00:51:19,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679885 2024-12-07T00:51:19,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679885 2024-12-07T00:51:19,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679885) roll requested 2024-12-07T00:51:19,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679895 2024-12-07T00:51:19,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679895 2024-12-07T00:51:19,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679895) roll requested 2024-12-07T00:51:19,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679900 2024-12-07T00:51:19,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679900 2024-12-07T00:51:19,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679900) roll requested 2024-12-07T00:51:19,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679907 2024-12-07T00:51:19,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679907 2024-12-07T00:51:19,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679907) roll requested 2024-12-07T00:51:19,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679915 2024-12-07T00:51:19,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679915 2024-12-07T00:51:19,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679915) roll requested 2024-12-07T00:51:19,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679922 2024-12-07T00:51:19,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679922 2024-12-07T00:51:19,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679922) roll requested 2024-12-07T00:51:19,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679932 2024-12-07T00:51:19,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679932 2024-12-07T00:51:19,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679932) roll requested 2024-12-07T00:51:19,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679940 2024-12-07T00:51:19,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679940 2024-12-07T00:51:19,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679940) roll requested 2024-12-07T00:51:19,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679949 2024-12-07T00:51:19,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679949 2024-12-07T00:51:19,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679949) roll requested 2024-12-07T00:51:19,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679957 2024-12-07T00:51:19,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679957 2024-12-07T00:51:19,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679957) roll requested 2024-12-07T00:51:19,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679965 2024-12-07T00:51:19,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679965 2024-12-07T00:51:19,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679965) roll requested 2024-12-07T00:51:19,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679974 2024-12-07T00:51:19,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679974 2024-12-07T00:51:19,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679974) roll requested 2024-12-07T00:51:19,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679981 2024-12-07T00:51:19,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679981 2024-12-07T00:51:19,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679981) roll requested 2024-12-07T00:51:19,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679987 2024-12-07T00:51:19,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679987 2024-12-07T00:51:19,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679987) roll requested 2024-12-07T00:51:19,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679993 2024-12-07T00:51:19,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:19,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679993 2024-12-07T00:51:19,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:19,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:19,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679993) roll requested 2024-12-07T00:51:20,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532679999 2024-12-07T00:51:20,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679999 2024-12-07T00:51:20,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532679999) roll requested 2024-12-07T00:51:20,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680008 2024-12-07T00:51:20,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532679999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680008 2024-12-07T00:51:20,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680008) roll requested 2024-12-07T00:51:20,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680017 2024-12-07T00:51:20,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680017 2024-12-07T00:51:20,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680017) roll requested 2024-12-07T00:51:20,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680026 2024-12-07T00:51:20,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680026 2024-12-07T00:51:20,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680026) roll requested 2024-12-07T00:51:20,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680036 2024-12-07T00:51:20,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680036 2024-12-07T00:51:20,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680036) roll requested 2024-12-07T00:51:20,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680045 2024-12-07T00:51:20,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680045 2024-12-07T00:51:20,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680045) roll requested 2024-12-07T00:51:20,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680055 2024-12-07T00:51:20,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680055 2024-12-07T00:51:20,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680055) roll requested 2024-12-07T00:51:20,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680064 2024-12-07T00:51:20,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680064 2024-12-07T00:51:20,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680064) roll requested 2024-12-07T00:51:20,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680073 2024-12-07T00:51:20,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680073 2024-12-07T00:51:20,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680073) roll requested 2024-12-07T00:51:20,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680079 2024-12-07T00:51:20,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680079 2024-12-07T00:51:20,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680079) roll requested 2024-12-07T00:51:20,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680087 2024-12-07T00:51:20,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680087 2024-12-07T00:51:20,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680087) roll requested 2024-12-07T00:51:20,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680094 2024-12-07T00:51:20,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680094 2024-12-07T00:51:20,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680094) roll requested 2024-12-07T00:51:20,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680101 2024-12-07T00:51:20,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680101 2024-12-07T00:51:20,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680101) roll requested 2024-12-07T00:51:20,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680108 2024-12-07T00:51:20,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680108 2024-12-07T00:51:20,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680108) roll requested 2024-12-07T00:51:20,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680117 2024-12-07T00:51:20,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680117 2024-12-07T00:51:20,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680117) roll requested 2024-12-07T00:51:20,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680127 2024-12-07T00:51:20,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680127 2024-12-07T00:51:20,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680127) roll requested 2024-12-07T00:51:20,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680138 2024-12-07T00:51:20,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680138 2024-12-07T00:51:20,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680138) roll requested 2024-12-07T00:51:20,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680147 2024-12-07T00:51:20,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680147 2024-12-07T00:51:20,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680147) roll requested 2024-12-07T00:51:20,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680152 2024-12-07T00:51:20,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680152 2024-12-07T00:51:20,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680152) roll requested 2024-12-07T00:51:20,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680162 2024-12-07T00:51:20,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680162 2024-12-07T00:51:20,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680162) roll requested 2024-12-07T00:51:20,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680171 2024-12-07T00:51:20,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680171 2024-12-07T00:51:20,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680171) roll requested 2024-12-07T00:51:20,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680177 2024-12-07T00:51:20,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680177 2024-12-07T00:51:20,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680177) roll requested 2024-12-07T00:51:20,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680182 2024-12-07T00:51:20,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680182 2024-12-07T00:51:20,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680182) roll requested 2024-12-07T00:51:20,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680190 2024-12-07T00:51:20,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680190 2024-12-07T00:51:20,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680190) roll requested 2024-12-07T00:51:20,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680198 2024-12-07T00:51:20,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680198 2024-12-07T00:51:20,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680198) roll requested 2024-12-07T00:51:20,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680204 2024-12-07T00:51:20,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680204 2024-12-07T00:51:20,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680204) roll requested 2024-12-07T00:51:20,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680211 2024-12-07T00:51:20,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680211 2024-12-07T00:51:20,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680211) roll requested 2024-12-07T00:51:20,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680221 2024-12-07T00:51:20,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680221 2024-12-07T00:51:20,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680221) roll requested 2024-12-07T00:51:20,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680230 2024-12-07T00:51:20,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680230 2024-12-07T00:51:20,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680230) roll requested 2024-12-07T00:51:20,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680239 2024-12-07T00:51:20,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680239 2024-12-07T00:51:20,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680239) roll requested 2024-12-07T00:51:20,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680249 2024-12-07T00:51:20,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680249 2024-12-07T00:51:20,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680249) roll requested 2024-12-07T00:51:20,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680258 2024-12-07T00:51:20,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680258 2024-12-07T00:51:20,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680258) roll requested 2024-12-07T00:51:20,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680268 2024-12-07T00:51:20,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680268 2024-12-07T00:51:20,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680268) roll requested 2024-12-07T00:51:20,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680278 2024-12-07T00:51:20,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680278 2024-12-07T00:51:20,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680278) roll requested 2024-12-07T00:51:20,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680287 2024-12-07T00:51:20,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680287 2024-12-07T00:51:20,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680287) roll requested 2024-12-07T00:51:20,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680294 2024-12-07T00:51:20,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680294 2024-12-07T00:51:20,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680294) roll requested 2024-12-07T00:51:20,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680300 2024-12-07T00:51:20,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680300 2024-12-07T00:51:20,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680300) roll requested 2024-12-07T00:51:20,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680308 2024-12-07T00:51:20,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680308 2024-12-07T00:51:20,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680308) roll requested 2024-12-07T00:51:20,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680315 2024-12-07T00:51:20,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680315 2024-12-07T00:51:20,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680315) roll requested 2024-12-07T00:51:20,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680320 2024-12-07T00:51:20,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680320 2024-12-07T00:51:20,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680320) roll requested 2024-12-07T00:51:20,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680326 2024-12-07T00:51:20,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680326 2024-12-07T00:51:20,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680326) roll requested 2024-12-07T00:51:20,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680334 2024-12-07T00:51:20,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680334 2024-12-07T00:51:20,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680334) roll requested 2024-12-07T00:51:20,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680342 2024-12-07T00:51:20,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680342 2024-12-07T00:51:20,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680342) roll requested 2024-12-07T00:51:20,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680348 2024-12-07T00:51:20,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680348 2024-12-07T00:51:20,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680348) roll requested 2024-12-07T00:51:20,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680353 2024-12-07T00:51:20,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680353 2024-12-07T00:51:20,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680353) roll requested 2024-12-07T00:51:20,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680361 2024-12-07T00:51:20,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680361 2024-12-07T00:51:20,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680361) roll requested 2024-12-07T00:51:20,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680367 2024-12-07T00:51:20,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680367 2024-12-07T00:51:20,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680367) roll requested 2024-12-07T00:51:20,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680372 2024-12-07T00:51:20,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680372 2024-12-07T00:51:20,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680372) roll requested 2024-12-07T00:51:20,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680378 2024-12-07T00:51:20,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680378 2024-12-07T00:51:20,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680378) roll requested 2024-12-07T00:51:20,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680383 2024-12-07T00:51:20,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680383 2024-12-07T00:51:20,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680383) roll requested 2024-12-07T00:51:20,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680390 2024-12-07T00:51:20,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680390 2024-12-07T00:51:20,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680390) roll requested 2024-12-07T00:51:20,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680396 2024-12-07T00:51:20,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680396 2024-12-07T00:51:20,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680396) roll requested 2024-12-07T00:51:20,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680404 2024-12-07T00:51:20,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680404 2024-12-07T00:51:20,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680404) roll requested 2024-12-07T00:51:20,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680412 2024-12-07T00:51:20,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680412 2024-12-07T00:51:20,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680412) roll requested 2024-12-07T00:51:20,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680420 2024-12-07T00:51:20,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680420 2024-12-07T00:51:20,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680420) roll requested 2024-12-07T00:51:20,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680427 2024-12-07T00:51:20,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680427 2024-12-07T00:51:20,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680427) roll requested 2024-12-07T00:51:20,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680435 2024-12-07T00:51:20,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680435 2024-12-07T00:51:20,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680435) roll requested 2024-12-07T00:51:20,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680441 2024-12-07T00:51:20,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680441 2024-12-07T00:51:20,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680441) roll requested 2024-12-07T00:51:20,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680447 2024-12-07T00:51:20,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680447 2024-12-07T00:51:20,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680447) roll requested 2024-12-07T00:51:20,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680452 2024-12-07T00:51:20,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680452 2024-12-07T00:51:20,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680452) roll requested 2024-12-07T00:51:20,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680458 2024-12-07T00:51:20,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680458 2024-12-07T00:51:20,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680458) roll requested 2024-12-07T00:51:20,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680463 2024-12-07T00:51:20,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680463 2024-12-07T00:51:20,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680463) roll requested 2024-12-07T00:51:20,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680469 2024-12-07T00:51:20,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680469 2024-12-07T00:51:20,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680469) roll requested 2024-12-07T00:51:20,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680474 2024-12-07T00:51:20,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680474 2024-12-07T00:51:20,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680474) roll requested 2024-12-07T00:51:20,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680479 2024-12-07T00:51:20,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680479 2024-12-07T00:51:20,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680479) roll requested 2024-12-07T00:51:20,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680486 2024-12-07T00:51:20,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680486 2024-12-07T00:51:20,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680486) roll requested 2024-12-07T00:51:20,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680494 2024-12-07T00:51:20,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680494 2024-12-07T00:51:20,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680494) roll requested 2024-12-07T00:51:20,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680502 2024-12-07T00:51:20,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680502 2024-12-07T00:51:20,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680502) roll requested 2024-12-07T00:51:20,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680510 2024-12-07T00:51:20,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680510 2024-12-07T00:51:20,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680510) roll requested 2024-12-07T00:51:20,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680516 2024-12-07T00:51:20,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680516 2024-12-07T00:51:20,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680516) roll requested 2024-12-07T00:51:20,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680523 2024-12-07T00:51:20,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680523 2024-12-07T00:51:20,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680523) roll requested 2024-12-07T00:51:20,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680528 2024-12-07T00:51:20,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680528 2024-12-07T00:51:20,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680528) roll requested 2024-12-07T00:51:20,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680534 2024-12-07T00:51:20,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680534 2024-12-07T00:51:20,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680534) roll requested 2024-12-07T00:51:20,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680539 2024-12-07T00:51:20,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680539 2024-12-07T00:51:20,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680539) roll requested 2024-12-07T00:51:20,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680545 2024-12-07T00:51:20,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680545 2024-12-07T00:51:20,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680545) roll requested 2024-12-07T00:51:20,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680551 2024-12-07T00:51:20,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680551 2024-12-07T00:51:20,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680551) roll requested 2024-12-07T00:51:20,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680559 2024-12-07T00:51:20,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680559 2024-12-07T00:51:20,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680559) roll requested 2024-12-07T00:51:20,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680568 2024-12-07T00:51:20,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680568 2024-12-07T00:51:20,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680568) roll requested 2024-12-07T00:51:20,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680577 2024-12-07T00:51:20,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680577 2024-12-07T00:51:20,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680577) roll requested 2024-12-07T00:51:20,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680585 2024-12-07T00:51:20,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680585 2024-12-07T00:51:20,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680585) roll requested 2024-12-07T00:51:20,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680594 2024-12-07T00:51:20,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680594 2024-12-07T00:51:20,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680594) roll requested 2024-12-07T00:51:20,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680599 2024-12-07T00:51:20,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680599 2024-12-07T00:51:20,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680599) roll requested 2024-12-07T00:51:20,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680604 2024-12-07T00:51:20,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680604 2024-12-07T00:51:20,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680604) roll requested 2024-12-07T00:51:20,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680611 2024-12-07T00:51:20,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680611 2024-12-07T00:51:20,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680611) roll requested 2024-12-07T00:51:20,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680618 2024-12-07T00:51:20,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680618 2024-12-07T00:51:20,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680618) roll requested 2024-12-07T00:51:20,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680624 2024-12-07T00:51:20,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680624 2024-12-07T00:51:20,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680624) roll requested 2024-12-07T00:51:20,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680630 2024-12-07T00:51:20,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680630 2024-12-07T00:51:20,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680630) roll requested 2024-12-07T00:51:20,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680636 2024-12-07T00:51:20,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680636 2024-12-07T00:51:20,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680636) roll requested 2024-12-07T00:51:20,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680644 2024-12-07T00:51:20,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680644 2024-12-07T00:51:20,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680644) roll requested 2024-12-07T00:51:20,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680652 2024-12-07T00:51:20,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680652 2024-12-07T00:51:20,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680652) roll requested 2024-12-07T00:51:20,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680661 2024-12-07T00:51:20,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680661 2024-12-07T00:51:20,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680661) roll requested 2024-12-07T00:51:20,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680671 2024-12-07T00:51:20,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680671 2024-12-07T00:51:20,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680671) roll requested 2024-12-07T00:51:20,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680680 2024-12-07T00:51:20,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680680 2024-12-07T00:51:20,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680680) roll requested 2024-12-07T00:51:20,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680689 2024-12-07T00:51:20,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680689 2024-12-07T00:51:20,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680689) roll requested 2024-12-07T00:51:20,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680697 2024-12-07T00:51:20,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680697 2024-12-07T00:51:20,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680697) roll requested 2024-12-07T00:51:20,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680707 2024-12-07T00:51:20,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680707 2024-12-07T00:51:20,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680707) roll requested 2024-12-07T00:51:20,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680716 2024-12-07T00:51:20,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680716 2024-12-07T00:51:20,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680716) roll requested 2024-12-07T00:51:20,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680725 2024-12-07T00:51:20,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680725 2024-12-07T00:51:20,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680725) roll requested 2024-12-07T00:51:20,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680735 2024-12-07T00:51:20,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680735 2024-12-07T00:51:20,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680735) roll requested 2024-12-07T00:51:20,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680745 2024-12-07T00:51:20,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680745 2024-12-07T00:51:20,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680745) roll requested 2024-12-07T00:51:20,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680751 2024-12-07T00:51:20,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680751 2024-12-07T00:51:20,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680751) roll requested 2024-12-07T00:51:20,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680757 2024-12-07T00:51:20,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680757 2024-12-07T00:51:20,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680757) roll requested 2024-12-07T00:51:20,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680768 2024-12-07T00:51:20,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680768 2024-12-07T00:51:20,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680768) roll requested 2024-12-07T00:51:20,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680776 2024-12-07T00:51:20,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680776 2024-12-07T00:51:20,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680776) roll requested 2024-12-07T00:51:20,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680782 2024-12-07T00:51:20,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680782 2024-12-07T00:51:20,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680782) roll requested 2024-12-07T00:51:20,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680788 2024-12-07T00:51:20,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680788 2024-12-07T00:51:20,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680788) roll requested 2024-12-07T00:51:20,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680793 2024-12-07T00:51:20,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680793 2024-12-07T00:51:20,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680793) roll requested 2024-12-07T00:51:20,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680801 2024-12-07T00:51:20,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680801 2024-12-07T00:51:20,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680801) roll requested 2024-12-07T00:51:20,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680806 2024-12-07T00:51:20,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680806 2024-12-07T00:51:20,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680801 is not closed yet, will try archiving it next time 2024-12-07T00:51:20,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680806) roll requested 2024-12-07T00:51:20,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680813 2024-12-07T00:51:20,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680813 2024-12-07T00:51:20,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680813) roll requested 2024-12-07T00:51:20,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680818 2024-12-07T00:51:20,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680818 2024-12-07T00:51:20,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680818) roll requested 2024-12-07T00:51:20,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680823 2024-12-07T00:51:20,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680823 2024-12-07T00:51:20,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680823) roll requested 2024-12-07T00:51:20,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680828 2024-12-07T00:51:20,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680828 2024-12-07T00:51:20,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680828) roll requested 2024-12-07T00:51:20,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680834 2024-12-07T00:51:20,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680834 2024-12-07T00:51:20,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680834) roll requested 2024-12-07T00:51:20,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680842 2024-12-07T00:51:20,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680842 2024-12-07T00:51:20,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680842) roll requested 2024-12-07T00:51:20,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680850 2024-12-07T00:51:20,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680850 2024-12-07T00:51:20,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680850) roll requested 2024-12-07T00:51:20,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680859 2024-12-07T00:51:20,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680859 2024-12-07T00:51:20,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680859) roll requested 2024-12-07T00:51:20,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680868 2024-12-07T00:51:20,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680868 2024-12-07T00:51:20,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680868) roll requested 2024-12-07T00:51:20,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680876 2024-12-07T00:51:20,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680876 2024-12-07T00:51:20,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680876) roll requested 2024-12-07T00:51:20,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680882 2024-12-07T00:51:20,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680882 2024-12-07T00:51:20,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680882) roll requested 2024-12-07T00:51:20,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680889 2024-12-07T00:51:20,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680889 2024-12-07T00:51:20,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680889) roll requested 2024-12-07T00:51:20,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680898 2024-12-07T00:51:20,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680898 2024-12-07T00:51:20,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680898) roll requested 2024-12-07T00:51:20,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680907 2024-12-07T00:51:20,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680907 2024-12-07T00:51:20,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680907) roll requested 2024-12-07T00:51:20,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680917 2024-12-07T00:51:20,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680917 2024-12-07T00:51:20,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680917) roll requested 2024-12-07T00:51:20,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680926 2024-12-07T00:51:20,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680926 2024-12-07T00:51:20,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680926) roll requested 2024-12-07T00:51:20,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680936 2024-12-07T00:51:20,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680936 2024-12-07T00:51:20,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680936) roll requested 2024-12-07T00:51:20,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680946 2024-12-07T00:51:20,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680946 2024-12-07T00:51:20,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680946) roll requested 2024-12-07T00:51:20,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680955 2024-12-07T00:51:20,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680955 2024-12-07T00:51:20,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680955) roll requested 2024-12-07T00:51:20,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680964 2024-12-07T00:51:20,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680964 2024-12-07T00:51:20,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680964) roll requested 2024-12-07T00:51:20,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680973 2024-12-07T00:51:20,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680973 2024-12-07T00:51:20,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680973) roll requested 2024-12-07T00:51:20,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680984 2024-12-07T00:51:20,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680984 2024-12-07T00:51:20,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680984) roll requested 2024-12-07T00:51:20,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680992 2024-12-07T00:51:20,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:20,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680992 2024-12-07T00:51:20,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:20,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:20,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680992) roll requested 2024-12-07T00:51:20,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532680998 2024-12-07T00:51:21,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680998 2024-12-07T00:51:21,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532680998) roll requested 2024-12-07T00:51:21,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681004 2024-12-07T00:51:21,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532680998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681004 2024-12-07T00:51:21,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681004) roll requested 2024-12-07T00:51:21,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681010 2024-12-07T00:51:21,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681010 2024-12-07T00:51:21,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681010) roll requested 2024-12-07T00:51:21,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681018 2024-12-07T00:51:21,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681018 2024-12-07T00:51:21,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681018) roll requested 2024-12-07T00:51:21,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681027 2024-12-07T00:51:21,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681027 2024-12-07T00:51:21,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681027) roll requested 2024-12-07T00:51:21,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681037 2024-12-07T00:51:21,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681037 2024-12-07T00:51:21,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681037) roll requested 2024-12-07T00:51:21,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681044 2024-12-07T00:51:21,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681044 2024-12-07T00:51:21,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681044) roll requested 2024-12-07T00:51:21,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681053 2024-12-07T00:51:21,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681053 2024-12-07T00:51:21,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681053) roll requested 2024-12-07T00:51:21,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681068 2024-12-07T00:51:21,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681068 2024-12-07T00:51:21,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681068) roll requested 2024-12-07T00:51:21,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681074 2024-12-07T00:51:21,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681074 2024-12-07T00:51:21,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681074) roll requested 2024-12-07T00:51:21,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681080 2024-12-07T00:51:21,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681080 2024-12-07T00:51:21,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681080) roll requested 2024-12-07T00:51:21,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681085 2024-12-07T00:51:21,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681085 2024-12-07T00:51:21,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681085) roll requested 2024-12-07T00:51:21,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681091 2024-12-07T00:51:21,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681091 2024-12-07T00:51:21,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681091) roll requested 2024-12-07T00:51:21,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681099 2024-12-07T00:51:21,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681099 2024-12-07T00:51:21,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681099) roll requested 2024-12-07T00:51:21,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681106 2024-12-07T00:51:21,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681106 2024-12-07T00:51:21,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681106) roll requested 2024-12-07T00:51:21,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681116 2024-12-07T00:51:21,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681116 2024-12-07T00:51:21,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681116) roll requested 2024-12-07T00:51:21,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681124 2024-12-07T00:51:21,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681124 2024-12-07T00:51:21,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681124) roll requested 2024-12-07T00:51:21,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681133 2024-12-07T00:51:21,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681133 2024-12-07T00:51:21,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681133) roll requested 2024-12-07T00:51:21,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681142 2024-12-07T00:51:21,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681142 2024-12-07T00:51:21,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681142) roll requested 2024-12-07T00:51:21,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681148 2024-12-07T00:51:21,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681148 2024-12-07T00:51:21,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681148) roll requested 2024-12-07T00:51:21,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681154 2024-12-07T00:51:21,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681154 2024-12-07T00:51:21,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681154) roll requested 2024-12-07T00:51:21,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681160 2024-12-07T00:51:21,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681160 2024-12-07T00:51:21,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681160) roll requested 2024-12-07T00:51:21,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681168 2024-12-07T00:51:21,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681168 2024-12-07T00:51:21,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681168) roll requested 2024-12-07T00:51:21,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681174 2024-12-07T00:51:21,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681174 2024-12-07T00:51:21,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681174) roll requested 2024-12-07T00:51:21,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681180 2024-12-07T00:51:21,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681180 2024-12-07T00:51:21,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681180) roll requested 2024-12-07T00:51:21,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681185 2024-12-07T00:51:21,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681185 2024-12-07T00:51:21,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681185) roll requested 2024-12-07T00:51:21,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681191 2024-12-07T00:51:21,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681191 2024-12-07T00:51:21,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681191) roll requested 2024-12-07T00:51:21,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681197 2024-12-07T00:51:21,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681197 2024-12-07T00:51:21,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681197) roll requested 2024-12-07T00:51:21,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681204 2024-12-07T00:51:21,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681204 2024-12-07T00:51:21,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681204) roll requested 2024-12-07T00:51:21,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681211 2024-12-07T00:51:21,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681211 2024-12-07T00:51:21,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681211) roll requested 2024-12-07T00:51:21,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681219 2024-12-07T00:51:21,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681219 2024-12-07T00:51:21,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681219) roll requested 2024-12-07T00:51:21,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681224 2024-12-07T00:51:21,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681224 2024-12-07T00:51:21,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681224) roll requested 2024-12-07T00:51:21,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681230 2024-12-07T00:51:21,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681230 2024-12-07T00:51:21,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681230) roll requested 2024-12-07T00:51:21,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681240 2024-12-07T00:51:21,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681240 2024-12-07T00:51:21,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681240) roll requested 2024-12-07T00:51:21,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681247 2024-12-07T00:51:21,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681247 2024-12-07T00:51:21,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681247) roll requested 2024-12-07T00:51:21,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681255 2024-12-07T00:51:21,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681255 2024-12-07T00:51:21,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681255) roll requested 2024-12-07T00:51:21,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681263 2024-12-07T00:51:21,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681263 2024-12-07T00:51:21,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681263) roll requested 2024-12-07T00:51:21,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681272 2024-12-07T00:51:21,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681272 2024-12-07T00:51:21,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681272) roll requested 2024-12-07T00:51:21,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681280 2024-12-07T00:51:21,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681280 2024-12-07T00:51:21,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681280) roll requested 2024-12-07T00:51:21,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681288 2024-12-07T00:51:21,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681288 2024-12-07T00:51:21,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681288) roll requested 2024-12-07T00:51:21,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681294 2024-12-07T00:51:21,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681294 2024-12-07T00:51:21,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681294) roll requested 2024-12-07T00:51:21,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681304 2024-12-07T00:51:21,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681304 2024-12-07T00:51:21,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681304) roll requested 2024-12-07T00:51:21,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681314 2024-12-07T00:51:21,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681314 2024-12-07T00:51:21,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681314) roll requested 2024-12-07T00:51:21,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681322 2024-12-07T00:51:21,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681322 2024-12-07T00:51:21,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681322) roll requested 2024-12-07T00:51:21,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681327 2024-12-07T00:51:21,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681327 2024-12-07T00:51:21,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681327) roll requested 2024-12-07T00:51:21,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681332 2024-12-07T00:51:21,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681332 2024-12-07T00:51:21,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681332) roll requested 2024-12-07T00:51:21,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681338 2024-12-07T00:51:21,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681338 2024-12-07T00:51:21,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681338) roll requested 2024-12-07T00:51:21,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681348 2024-12-07T00:51:21,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681348 2024-12-07T00:51:21,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681348) roll requested 2024-12-07T00:51:21,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681355 2024-12-07T00:51:21,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681355 2024-12-07T00:51:21,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681355) roll requested 2024-12-07T00:51:21,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681360 2024-12-07T00:51:21,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681360 2024-12-07T00:51:21,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681360) roll requested 2024-12-07T00:51:21,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681366 2024-12-07T00:51:21,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681366 2024-12-07T00:51:21,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681366) roll requested 2024-12-07T00:51:21,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681374 2024-12-07T00:51:21,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681374 2024-12-07T00:51:21,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681374) roll requested 2024-12-07T00:51:21,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681382 2024-12-07T00:51:21,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681382 2024-12-07T00:51:21,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681382) roll requested 2024-12-07T00:51:21,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681391 2024-12-07T00:51:21,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681391 2024-12-07T00:51:21,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681391) roll requested 2024-12-07T00:51:21,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681399 2024-12-07T00:51:21,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681399 2024-12-07T00:51:21,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681399) roll requested 2024-12-07T00:51:21,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681408 2024-12-07T00:51:21,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681408 2024-12-07T00:51:21,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681408) roll requested 2024-12-07T00:51:21,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681415 2024-12-07T00:51:21,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681415 2024-12-07T00:51:21,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681415) roll requested 2024-12-07T00:51:21,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681423 2024-12-07T00:51:21,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681423 2024-12-07T00:51:21,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681423) roll requested 2024-12-07T00:51:21,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681432 2024-12-07T00:51:21,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681432 2024-12-07T00:51:21,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681432) roll requested 2024-12-07T00:51:21,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681441 2024-12-07T00:51:21,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681441 2024-12-07T00:51:21,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681441) roll requested 2024-12-07T00:51:21,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681451 2024-12-07T00:51:21,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681451 2024-12-07T00:51:21,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681451) roll requested 2024-12-07T00:51:21,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681459 2024-12-07T00:51:21,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681459 2024-12-07T00:51:21,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681459) roll requested 2024-12-07T00:51:21,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681467 2024-12-07T00:51:21,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681467 2024-12-07T00:51:21,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681467) roll requested 2024-12-07T00:51:21,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681475 2024-12-07T00:51:21,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681475 2024-12-07T00:51:21,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681475) roll requested 2024-12-07T00:51:21,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681484 2024-12-07T00:51:21,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681484 2024-12-07T00:51:21,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681484) roll requested 2024-12-07T00:51:21,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681490 2024-12-07T00:51:21,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681490 2024-12-07T00:51:21,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681490) roll requested 2024-12-07T00:51:21,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681495 2024-12-07T00:51:21,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681495 2024-12-07T00:51:21,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681495) roll requested 2024-12-07T00:51:21,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681500 2024-12-07T00:51:21,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681500 2024-12-07T00:51:21,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681500) roll requested 2024-12-07T00:51:21,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681509 2024-12-07T00:51:21,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681509 2024-12-07T00:51:21,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681509) roll requested 2024-12-07T00:51:21,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681518 2024-12-07T00:51:21,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681518 2024-12-07T00:51:21,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681518) roll requested 2024-12-07T00:51:21,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681525 2024-12-07T00:51:21,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681525 2024-12-07T00:51:21,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681525) roll requested 2024-12-07T00:51:21,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681532 2024-12-07T00:51:21,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681532 2024-12-07T00:51:21,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681532) roll requested 2024-12-07T00:51:21,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681541 2024-12-07T00:51:21,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681541 2024-12-07T00:51:21,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681541) roll requested 2024-12-07T00:51:21,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681550 2024-12-07T00:51:21,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681550 2024-12-07T00:51:21,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681550) roll requested 2024-12-07T00:51:21,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681559 2024-12-07T00:51:21,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681559 2024-12-07T00:51:21,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681559) roll requested 2024-12-07T00:51:21,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681571 2024-12-07T00:51:21,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681571 2024-12-07T00:51:21,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681571) roll requested 2024-12-07T00:51:21,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681578 2024-12-07T00:51:21,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681578 2024-12-07T00:51:21,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681578) roll requested 2024-12-07T00:51:21,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681584 2024-12-07T00:51:21,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681584 2024-12-07T00:51:21,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681584) roll requested 2024-12-07T00:51:21,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681590 2024-12-07T00:51:21,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681590 2024-12-07T00:51:21,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681590) roll requested 2024-12-07T00:51:21,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681597 2024-12-07T00:51:21,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681597 2024-12-07T00:51:21,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681597) roll requested 2024-12-07T00:51:21,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681605 2024-12-07T00:51:21,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681605 2024-12-07T00:51:21,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681605) roll requested 2024-12-07T00:51:21,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681614 2024-12-07T00:51:21,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681614 2024-12-07T00:51:21,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681614) roll requested 2024-12-07T00:51:21,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681619 2024-12-07T00:51:21,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681619 2024-12-07T00:51:21,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681619) roll requested 2024-12-07T00:51:21,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681624 2024-12-07T00:51:21,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681624 2024-12-07T00:51:21,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681624) roll requested 2024-12-07T00:51:21,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681631 2024-12-07T00:51:21,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681631 2024-12-07T00:51:21,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681631) roll requested 2024-12-07T00:51:21,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681639 2024-12-07T00:51:21,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681639 2024-12-07T00:51:21,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681639) roll requested 2024-12-07T00:51:21,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681648 2024-12-07T00:51:21,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681648 2024-12-07T00:51:21,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681648) roll requested 2024-12-07T00:51:21,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681657 2024-12-07T00:51:21,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681657 2024-12-07T00:51:21,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681657) roll requested 2024-12-07T00:51:21,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681668 2024-12-07T00:51:21,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681668 2024-12-07T00:51:21,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681668) roll requested 2024-12-07T00:51:21,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681679 2024-12-07T00:51:21,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681679 2024-12-07T00:51:21,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681679) roll requested 2024-12-07T00:51:21,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681689 2024-12-07T00:51:21,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681689 2024-12-07T00:51:21,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681689) roll requested 2024-12-07T00:51:21,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681695 2024-12-07T00:51:21,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681695 2024-12-07T00:51:21,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681695) roll requested 2024-12-07T00:51:21,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681701 2024-12-07T00:51:21,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681701 2024-12-07T00:51:21,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681701) roll requested 2024-12-07T00:51:21,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681707 2024-12-07T00:51:21,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681707 2024-12-07T00:51:21,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681707) roll requested 2024-12-07T00:51:21,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681715 2024-12-07T00:51:21,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681715 2024-12-07T00:51:21,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681715) roll requested 2024-12-07T00:51:21,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681724 2024-12-07T00:51:21,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681724 2024-12-07T00:51:21,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681724) roll requested 2024-12-07T00:51:21,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681734 2024-12-07T00:51:21,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681734 2024-12-07T00:51:21,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681734) roll requested 2024-12-07T00:51:21,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681743 2024-12-07T00:51:21,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681743 2024-12-07T00:51:21,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681743) roll requested 2024-12-07T00:51:21,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681751 2024-12-07T00:51:21,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681751 2024-12-07T00:51:21,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681751) roll requested 2024-12-07T00:51:21,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681760 2024-12-07T00:51:21,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681760 2024-12-07T00:51:21,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681760) roll requested 2024-12-07T00:51:21,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681768 2024-12-07T00:51:21,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681768 2024-12-07T00:51:21,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681768) roll requested 2024-12-07T00:51:21,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681774 2024-12-07T00:51:21,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681774 2024-12-07T00:51:21,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681774) roll requested 2024-12-07T00:51:21,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681780 2024-12-07T00:51:21,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681780 2024-12-07T00:51:21,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681780) roll requested 2024-12-07T00:51:21,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681784 2024-12-07T00:51:21,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681784 2024-12-07T00:51:21,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681784) roll requested 2024-12-07T00:51:21,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681792 2024-12-07T00:51:21,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681792 2024-12-07T00:51:21,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681792) roll requested 2024-12-07T00:51:21,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681798 2024-12-07T00:51:21,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681798 2024-12-07T00:51:21,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681798) roll requested 2024-12-07T00:51:21,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681806 2024-12-07T00:51:21,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681806 2024-12-07T00:51:21,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681806) roll requested 2024-12-07T00:51:21,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681814 2024-12-07T00:51:21,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681814 2024-12-07T00:51:21,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681814) roll requested 2024-12-07T00:51:21,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681820 2024-12-07T00:51:21,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681820 2024-12-07T00:51:21,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681820) roll requested 2024-12-07T00:51:21,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681826 2024-12-07T00:51:21,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681826 2024-12-07T00:51:21,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681826) roll requested 2024-12-07T00:51:21,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681832 2024-12-07T00:51:21,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681832 2024-12-07T00:51:21,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681832) roll requested 2024-12-07T00:51:21,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681837 2024-12-07T00:51:21,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681837 2024-12-07T00:51:21,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681837) roll requested 2024-12-07T00:51:21,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681843 2024-12-07T00:51:21,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681843 2024-12-07T00:51:21,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681843) roll requested 2024-12-07T00:51:21,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681849 2024-12-07T00:51:21,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681849 2024-12-07T00:51:21,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681849) roll requested 2024-12-07T00:51:21,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681855 2024-12-07T00:51:21,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681855 2024-12-07T00:51:21,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681855) roll requested 2024-12-07T00:51:21,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681865 2024-12-07T00:51:21,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681865 2024-12-07T00:51:21,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681865) roll requested 2024-12-07T00:51:21,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681875 2024-12-07T00:51:21,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681875 2024-12-07T00:51:21,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681875) roll requested 2024-12-07T00:51:21,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681881 2024-12-07T00:51:21,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681881 2024-12-07T00:51:21,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681881) roll requested 2024-12-07T00:51:21,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681887 2024-12-07T00:51:21,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681887 2024-12-07T00:51:21,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681887) roll requested 2024-12-07T00:51:21,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681893 2024-12-07T00:51:21,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681893 2024-12-07T00:51:21,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681893) roll requested 2024-12-07T00:51:21,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681898 2024-12-07T00:51:21,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681898 2024-12-07T00:51:21,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681898) roll requested 2024-12-07T00:51:21,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681903 2024-12-07T00:51:21,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681903 2024-12-07T00:51:21,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681903) roll requested 2024-12-07T00:51:21,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681909 2024-12-07T00:51:21,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681909 2024-12-07T00:51:21,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681909) roll requested 2024-12-07T00:51:21,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681915 2024-12-07T00:51:21,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681915 2024-12-07T00:51:21,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681915) roll requested 2024-12-07T00:51:21,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681921 2024-12-07T00:51:21,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681921 2024-12-07T00:51:21,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681921) roll requested 2024-12-07T00:51:21,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681927 2024-12-07T00:51:21,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681927 2024-12-07T00:51:21,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681927) roll requested 2024-12-07T00:51:21,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681934 2024-12-07T00:51:21,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681934 2024-12-07T00:51:21,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681934) roll requested 2024-12-07T00:51:21,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681939 2024-12-07T00:51:21,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681939 2024-12-07T00:51:21,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681939) roll requested 2024-12-07T00:51:21,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681944 2024-12-07T00:51:21,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681944 2024-12-07T00:51:21,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681944) roll requested 2024-12-07T00:51:21,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681950 2024-12-07T00:51:21,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681950 2024-12-07T00:51:21,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681950) roll requested 2024-12-07T00:51:21,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681955 2024-12-07T00:51:21,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681955 2024-12-07T00:51:21,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681955) roll requested 2024-12-07T00:51:21,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681961 2024-12-07T00:51:21,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681961 2024-12-07T00:51:21,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681961) roll requested 2024-12-07T00:51:21,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681967 2024-12-07T00:51:21,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681967 2024-12-07T00:51:21,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681967) roll requested 2024-12-07T00:51:21,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681972 2024-12-07T00:51:21,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681972 2024-12-07T00:51:21,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681972) roll requested 2024-12-07T00:51:21,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681978 2024-12-07T00:51:21,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681978 2024-12-07T00:51:21,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681978) roll requested 2024-12-07T00:51:21,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681984 2024-12-07T00:51:21,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681984 2024-12-07T00:51:21,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681984) roll requested 2024-12-07T00:51:21,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681989 2024-12-07T00:51:21,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:21,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681989 2024-12-07T00:51:21,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:21,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:21,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681989) roll requested 2024-12-07T00:51:21,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532681996 2024-12-07T00:51:22,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681996 2024-12-07T00:51:22,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532681996) roll requested 2024-12-07T00:51:22,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682001 2024-12-07T00:51:22,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532681996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682001 2024-12-07T00:51:22,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682001) roll requested 2024-12-07T00:51:22,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682008 2024-12-07T00:51:22,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682008 2024-12-07T00:51:22,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682008) roll requested 2024-12-07T00:51:22,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682013 2024-12-07T00:51:22,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682013 2024-12-07T00:51:22,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682013) roll requested 2024-12-07T00:51:22,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682019 2024-12-07T00:51:22,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682019 2024-12-07T00:51:22,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682019) roll requested 2024-12-07T00:51:22,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682025 2024-12-07T00:51:22,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682025 2024-12-07T00:51:22,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682025) roll requested 2024-12-07T00:51:22,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682031 2024-12-07T00:51:22,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682031 2024-12-07T00:51:22,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682031) roll requested 2024-12-07T00:51:22,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682036 2024-12-07T00:51:22,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682036 2024-12-07T00:51:22,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682036) roll requested 2024-12-07T00:51:22,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682041 2024-12-07T00:51:22,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682041 2024-12-07T00:51:22,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682041) roll requested 2024-12-07T00:51:22,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682047 2024-12-07T00:51:22,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682047 2024-12-07T00:51:22,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682047) roll requested 2024-12-07T00:51:22,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682053 2024-12-07T00:51:22,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682053 2024-12-07T00:51:22,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682053) roll requested 2024-12-07T00:51:22,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682059 2024-12-07T00:51:22,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682059 2024-12-07T00:51:22,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682059) roll requested 2024-12-07T00:51:22,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682065 2024-12-07T00:51:22,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682065 2024-12-07T00:51:22,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682065) roll requested 2024-12-07T00:51:22,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682071 2024-12-07T00:51:22,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682071 2024-12-07T00:51:22,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682071) roll requested 2024-12-07T00:51:22,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682076 2024-12-07T00:51:22,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682076 2024-12-07T00:51:22,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682076) roll requested 2024-12-07T00:51:22,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682082 2024-12-07T00:51:22,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682082 2024-12-07T00:51:22,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682082) roll requested 2024-12-07T00:51:22,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682088 2024-12-07T00:51:22,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682088 2024-12-07T00:51:22,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682088) roll requested 2024-12-07T00:51:22,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682094 2024-12-07T00:51:22,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682094 2024-12-07T00:51:22,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682094) roll requested 2024-12-07T00:51:22,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682100 2024-12-07T00:51:22,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682100 2024-12-07T00:51:22,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682100) roll requested 2024-12-07T00:51:22,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682106 2024-12-07T00:51:22,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682106 2024-12-07T00:51:22,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682106) roll requested 2024-12-07T00:51:22,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682111 2024-12-07T00:51:22,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682111 2024-12-07T00:51:22,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682111) roll requested 2024-12-07T00:51:22,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682117 2024-12-07T00:51:22,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682117 2024-12-07T00:51:22,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682117) roll requested 2024-12-07T00:51:22,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682123 2024-12-07T00:51:22,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682123 2024-12-07T00:51:22,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682123) roll requested 2024-12-07T00:51:22,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682129 2024-12-07T00:51:22,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682129 2024-12-07T00:51:22,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682129) roll requested 2024-12-07T00:51:22,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682135 2024-12-07T00:51:22,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682135 2024-12-07T00:51:22,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682135) roll requested 2024-12-07T00:51:22,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682140 2024-12-07T00:51:22,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682140 2024-12-07T00:51:22,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682140) roll requested 2024-12-07T00:51:22,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682145 2024-12-07T00:51:22,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682145 2024-12-07T00:51:22,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682145) roll requested 2024-12-07T00:51:22,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682152 2024-12-07T00:51:22,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682152 2024-12-07T00:51:22,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682152) roll requested 2024-12-07T00:51:22,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682160 2024-12-07T00:51:22,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682160 2024-12-07T00:51:22,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682160) roll requested 2024-12-07T00:51:22,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682168 2024-12-07T00:51:22,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682168 2024-12-07T00:51:22,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682168) roll requested 2024-12-07T00:51:22,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682174 2024-12-07T00:51:22,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682174 2024-12-07T00:51:22,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682174) roll requested 2024-12-07T00:51:22,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682182 2024-12-07T00:51:22,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682182 2024-12-07T00:51:22,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682182) roll requested 2024-12-07T00:51:22,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682190 2024-12-07T00:51:22,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682190 2024-12-07T00:51:22,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682190) roll requested 2024-12-07T00:51:22,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682197 2024-12-07T00:51:22,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682197 2024-12-07T00:51:22,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682197) roll requested 2024-12-07T00:51:22,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682204 2024-12-07T00:51:22,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682204 2024-12-07T00:51:22,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682204) roll requested 2024-12-07T00:51:22,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682213 2024-12-07T00:51:22,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682213 2024-12-07T00:51:22,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682213) roll requested 2024-12-07T00:51:22,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682225 2024-12-07T00:51:22,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682225 2024-12-07T00:51:22,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682225) roll requested 2024-12-07T00:51:22,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682231 2024-12-07T00:51:22,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682231 2024-12-07T00:51:22,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682231) roll requested 2024-12-07T00:51:22,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682237 2024-12-07T00:51:22,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682237 2024-12-07T00:51:22,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682237) roll requested 2024-12-07T00:51:22,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682245 2024-12-07T00:51:22,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682245 2024-12-07T00:51:22,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682245) roll requested 2024-12-07T00:51:22,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682252 2024-12-07T00:51:22,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682252 2024-12-07T00:51:22,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682252) roll requested 2024-12-07T00:51:22,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682260 2024-12-07T00:51:22,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682260 2024-12-07T00:51:22,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682260) roll requested 2024-12-07T00:51:22,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682268 2024-12-07T00:51:22,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682268 2024-12-07T00:51:22,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682268) roll requested 2024-12-07T00:51:22,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682276 2024-12-07T00:51:22,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682276 2024-12-07T00:51:22,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682276) roll requested 2024-12-07T00:51:22,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682287 2024-12-07T00:51:22,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682287 2024-12-07T00:51:22,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682287) roll requested 2024-12-07T00:51:22,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682293 2024-12-07T00:51:22,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682293 2024-12-07T00:51:22,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682293) roll requested 2024-12-07T00:51:22,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682300 2024-12-07T00:51:22,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682300 2024-12-07T00:51:22,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682300) roll requested 2024-12-07T00:51:22,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682308 2024-12-07T00:51:22,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682308 2024-12-07T00:51:22,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682308) roll requested 2024-12-07T00:51:22,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682313 2024-12-07T00:51:22,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682313 2024-12-07T00:51:22,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682313) roll requested 2024-12-07T00:51:22,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682319 2024-12-07T00:51:22,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682319 2024-12-07T00:51:22,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682319) roll requested 2024-12-07T00:51:22,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682327 2024-12-07T00:51:22,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682327 2024-12-07T00:51:22,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682327) roll requested 2024-12-07T00:51:22,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682333 2024-12-07T00:51:22,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682333 2024-12-07T00:51:22,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682333) roll requested 2024-12-07T00:51:22,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682344 2024-12-07T00:51:22,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682344 2024-12-07T00:51:22,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682344) roll requested 2024-12-07T00:51:22,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682352 2024-12-07T00:51:22,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682352 2024-12-07T00:51:22,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682352) roll requested 2024-12-07T00:51:22,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682358 2024-12-07T00:51:22,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682358 2024-12-07T00:51:22,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682358) roll requested 2024-12-07T00:51:22,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682366 2024-12-07T00:51:22,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682366 2024-12-07T00:51:22,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682366) roll requested 2024-12-07T00:51:22,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682374 2024-12-07T00:51:22,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682374 2024-12-07T00:51:22,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682374) roll requested 2024-12-07T00:51:22,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682379 2024-12-07T00:51:22,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682379 2024-12-07T00:51:22,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682379) roll requested 2024-12-07T00:51:22,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682385 2024-12-07T00:51:22,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682385 2024-12-07T00:51:22,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682385) roll requested 2024-12-07T00:51:22,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682392 2024-12-07T00:51:22,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682392 2024-12-07T00:51:22,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682392) roll requested 2024-12-07T00:51:22,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682400 2024-12-07T00:51:22,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682400 2024-12-07T00:51:22,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682400) roll requested 2024-12-07T00:51:22,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682406 2024-12-07T00:51:22,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682406 2024-12-07T00:51:22,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682406) roll requested 2024-12-07T00:51:22,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682411 2024-12-07T00:51:22,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682411 2024-12-07T00:51:22,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682411) roll requested 2024-12-07T00:51:22,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682417 2024-12-07T00:51:22,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682417 2024-12-07T00:51:22,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682417) roll requested 2024-12-07T00:51:22,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682422 2024-12-07T00:51:22,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682422 2024-12-07T00:51:22,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682422) roll requested 2024-12-07T00:51:22,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682431 2024-12-07T00:51:22,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682431 2024-12-07T00:51:22,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682431) roll requested 2024-12-07T00:51:22,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682439 2024-12-07T00:51:22,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682439 2024-12-07T00:51:22,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682439) roll requested 2024-12-07T00:51:22,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682447 2024-12-07T00:51:22,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682447 2024-12-07T00:51:22,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682447) roll requested 2024-12-07T00:51:22,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682453 2024-12-07T00:51:22,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682453 2024-12-07T00:51:22,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682453) roll requested 2024-12-07T00:51:22,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682459 2024-12-07T00:51:22,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682459 2024-12-07T00:51:22,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682459) roll requested 2024-12-07T00:51:22,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682465 2024-12-07T00:51:22,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682465 2024-12-07T00:51:22,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682465) roll requested 2024-12-07T00:51:22,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682472 2024-12-07T00:51:22,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682472 2024-12-07T00:51:22,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682472) roll requested 2024-12-07T00:51:22,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682481 2024-12-07T00:51:22,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682481 2024-12-07T00:51:22,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682481) roll requested 2024-12-07T00:51:22,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682488 2024-12-07T00:51:22,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682488 2024-12-07T00:51:22,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682488) roll requested 2024-12-07T00:51:22,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682496 2024-12-07T00:51:22,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682496 2024-12-07T00:51:22,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682496) roll requested 2024-12-07T00:51:22,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682504 2024-12-07T00:51:22,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682504 2024-12-07T00:51:22,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682504) roll requested 2024-12-07T00:51:22,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682512 2024-12-07T00:51:22,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682512 2024-12-07T00:51:22,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682512) roll requested 2024-12-07T00:51:22,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682519 2024-12-07T00:51:22,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682519 2024-12-07T00:51:22,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682519) roll requested 2024-12-07T00:51:22,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682524 2024-12-07T00:51:22,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682524 2024-12-07T00:51:22,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682524) roll requested 2024-12-07T00:51:22,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682533 2024-12-07T00:51:22,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682533 2024-12-07T00:51:22,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682533) roll requested 2024-12-07T00:51:22,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682539 2024-12-07T00:51:22,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682539 2024-12-07T00:51:22,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682539) roll requested 2024-12-07T00:51:22,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682545 2024-12-07T00:51:22,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682545 2024-12-07T00:51:22,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682545) roll requested 2024-12-07T00:51:22,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682552 2024-12-07T00:51:22,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682552 2024-12-07T00:51:22,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682552) roll requested 2024-12-07T00:51:22,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682560 2024-12-07T00:51:22,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682560 2024-12-07T00:51:22,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682560) roll requested 2024-12-07T00:51:22,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682566 2024-12-07T00:51:22,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682566 2024-12-07T00:51:22,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682566) roll requested 2024-12-07T00:51:22,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682572 2024-12-07T00:51:22,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682572 2024-12-07T00:51:22,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682572) roll requested 2024-12-07T00:51:22,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682580 2024-12-07T00:51:22,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682580 2024-12-07T00:51:22,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682580) roll requested 2024-12-07T00:51:22,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682587 2024-12-07T00:51:22,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682587 2024-12-07T00:51:22,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682587) roll requested 2024-12-07T00:51:22,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682592 2024-12-07T00:51:22,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682592 2024-12-07T00:51:22,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682592) roll requested 2024-12-07T00:51:22,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682597 2024-12-07T00:51:22,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682597 2024-12-07T00:51:22,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682597) roll requested 2024-12-07T00:51:22,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682602 2024-12-07T00:51:22,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682602 2024-12-07T00:51:22,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682602) roll requested 2024-12-07T00:51:22,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682608 2024-12-07T00:51:22,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682608 2024-12-07T00:51:22,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682608) roll requested 2024-12-07T00:51:22,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682614 2024-12-07T00:51:22,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682614 2024-12-07T00:51:22,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682614) roll requested 2024-12-07T00:51:22,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682622 2024-12-07T00:51:22,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682622 2024-12-07T00:51:22,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682622) roll requested 2024-12-07T00:51:22,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682631 2024-12-07T00:51:22,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682631 2024-12-07T00:51:22,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682631) roll requested 2024-12-07T00:51:22,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682637 2024-12-07T00:51:22,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682637 2024-12-07T00:51:22,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682637) roll requested 2024-12-07T00:51:22,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682646 2024-12-07T00:51:22,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682646 2024-12-07T00:51:22,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682646) roll requested 2024-12-07T00:51:22,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682654 2024-12-07T00:51:22,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682654 2024-12-07T00:51:22,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682654) roll requested 2024-12-07T00:51:22,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682660 2024-12-07T00:51:22,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682660 2024-12-07T00:51:22,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682660) roll requested 2024-12-07T00:51:22,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682667 2024-12-07T00:51:22,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682667 2024-12-07T00:51:22,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682667) roll requested 2024-12-07T00:51:22,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682674 2024-12-07T00:51:22,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682674 2024-12-07T00:51:22,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682674) roll requested 2024-12-07T00:51:22,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682683 2024-12-07T00:51:22,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682683 2024-12-07T00:51:22,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682683) roll requested 2024-12-07T00:51:22,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682688 2024-12-07T00:51:22,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682688 2024-12-07T00:51:22,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682688) roll requested 2024-12-07T00:51:22,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682693 2024-12-07T00:51:22,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682693 2024-12-07T00:51:22,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682693) roll requested 2024-12-07T00:51:22,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682699 2024-12-07T00:51:22,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682699 2024-12-07T00:51:22,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682699) roll requested 2024-12-07T00:51:22,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682707 2024-12-07T00:51:22,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682707 2024-12-07T00:51:22,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682707) roll requested 2024-12-07T00:51:22,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682715 2024-12-07T00:51:22,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682715 2024-12-07T00:51:22,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682715) roll requested 2024-12-07T00:51:22,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682723 2024-12-07T00:51:22,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682723 2024-12-07T00:51:22,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682723) roll requested 2024-12-07T00:51:22,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682731 2024-12-07T00:51:22,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682731 2024-12-07T00:51:22,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682731) roll requested 2024-12-07T00:51:22,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682740 2024-12-07T00:51:22,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682740 2024-12-07T00:51:22,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682740) roll requested 2024-12-07T00:51:22,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682750 2024-12-07T00:51:22,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682750 2024-12-07T00:51:22,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682750) roll requested 2024-12-07T00:51:22,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682759 2024-12-07T00:51:22,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682759 2024-12-07T00:51:22,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682759) roll requested 2024-12-07T00:51:22,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682767 2024-12-07T00:51:22,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682767 2024-12-07T00:51:22,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682767) roll requested 2024-12-07T00:51:22,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682774 2024-12-07T00:51:22,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682774 2024-12-07T00:51:22,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682774) roll requested 2024-12-07T00:51:22,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682781 2024-12-07T00:51:22,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682781 2024-12-07T00:51:22,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682781) roll requested 2024-12-07T00:51:22,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682790 2024-12-07T00:51:22,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682790 2024-12-07T00:51:22,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682790) roll requested 2024-12-07T00:51:22,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682798 2024-12-07T00:51:22,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682798 2024-12-07T00:51:22,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682798) roll requested 2024-12-07T00:51:22,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682806 2024-12-07T00:51:22,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682806 2024-12-07T00:51:22,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682806) roll requested 2024-12-07T00:51:22,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682815 2024-12-07T00:51:22,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682815 2024-12-07T00:51:22,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682815) roll requested 2024-12-07T00:51:22,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682825 2024-12-07T00:51:22,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682825 2024-12-07T00:51:22,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682825) roll requested 2024-12-07T00:51:22,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682834 2024-12-07T00:51:22,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682834 2024-12-07T00:51:22,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682834) roll requested 2024-12-07T00:51:22,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682842 2024-12-07T00:51:22,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682842 2024-12-07T00:51:22,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682842) roll requested 2024-12-07T00:51:22,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682851 2024-12-07T00:51:22,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682851 2024-12-07T00:51:22,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682851) roll requested 2024-12-07T00:51:22,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682860 2024-12-07T00:51:22,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682860 2024-12-07T00:51:22,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682860) roll requested 2024-12-07T00:51:22,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682867 2024-12-07T00:51:22,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682867 2024-12-07T00:51:22,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682867) roll requested 2024-12-07T00:51:22,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682875 2024-12-07T00:51:22,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682875 2024-12-07T00:51:22,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682875) roll requested 2024-12-07T00:51:22,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682883 2024-12-07T00:51:22,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682883 2024-12-07T00:51:22,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682883) roll requested 2024-12-07T00:51:22,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682890 2024-12-07T00:51:22,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682890 2024-12-07T00:51:22,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682890) roll requested 2024-12-07T00:51:22,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682898 2024-12-07T00:51:22,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682898 2024-12-07T00:51:22,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682898) roll requested 2024-12-07T00:51:22,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682907 2024-12-07T00:51:22,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682907 2024-12-07T00:51:22,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682907) roll requested 2024-12-07T00:51:22,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682917 2024-12-07T00:51:22,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682917 2024-12-07T00:51:22,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682917) roll requested 2024-12-07T00:51:22,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682924 2024-12-07T00:51:22,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682924 2024-12-07T00:51:22,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682924) roll requested 2024-12-07T00:51:22,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682931 2024-12-07T00:51:22,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682931 2024-12-07T00:51:22,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682931) roll requested 2024-12-07T00:51:22,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682939 2024-12-07T00:51:22,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682939 2024-12-07T00:51:22,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682939) roll requested 2024-12-07T00:51:22,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682948 2024-12-07T00:51:22,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682948 2024-12-07T00:51:22,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682948) roll requested 2024-12-07T00:51:22,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682956 2024-12-07T00:51:22,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682956 2024-12-07T00:51:22,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682956) roll requested 2024-12-07T00:51:22,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682965 2024-12-07T00:51:22,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682965 2024-12-07T00:51:22,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682965) roll requested 2024-12-07T00:51:22,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682973 2024-12-07T00:51:22,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682973 2024-12-07T00:51:22,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682973) roll requested 2024-12-07T00:51:22,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682982 2024-12-07T00:51:22,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682982 2024-12-07T00:51:22,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:22,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:22,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682982) roll requested 2024-12-07T00:51:22,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532682991 2024-12-07T00:51:22,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:22,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682991 2024-12-07T00:51:23,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532682991) roll requested 2024-12-07T00:51:23,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683000 2024-12-07T00:51:23,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532682991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683000 2024-12-07T00:51:23,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683000) roll requested 2024-12-07T00:51:23,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683008 2024-12-07T00:51:23,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683008 2024-12-07T00:51:23,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683008) roll requested 2024-12-07T00:51:23,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683014 2024-12-07T00:51:23,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683014 2024-12-07T00:51:23,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683014) roll requested 2024-12-07T00:51:23,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683021 2024-12-07T00:51:23,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683021 2024-12-07T00:51:23,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683021) roll requested 2024-12-07T00:51:23,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683028 2024-12-07T00:51:23,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683028 2024-12-07T00:51:23,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683028) roll requested 2024-12-07T00:51:23,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683038 2024-12-07T00:51:23,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683038 2024-12-07T00:51:23,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683038) roll requested 2024-12-07T00:51:23,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683047 2024-12-07T00:51:23,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683047 2024-12-07T00:51:23,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683047) roll requested 2024-12-07T00:51:23,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683056 2024-12-07T00:51:23,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683056 2024-12-07T00:51:23,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683056) roll requested 2024-12-07T00:51:23,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683064 2024-12-07T00:51:23,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683064 2024-12-07T00:51:23,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683064) roll requested 2024-12-07T00:51:23,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683070 2024-12-07T00:51:23,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683070 2024-12-07T00:51:23,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683070) roll requested 2024-12-07T00:51:23,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683076 2024-12-07T00:51:23,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683076 2024-12-07T00:51:23,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683076) roll requested 2024-12-07T00:51:23,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683088 2024-12-07T00:51:23,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683088 2024-12-07T00:51:23,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683088) roll requested 2024-12-07T00:51:23,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683094 2024-12-07T00:51:23,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683094 2024-12-07T00:51:23,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683094) roll requested 2024-12-07T00:51:23,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683100 2024-12-07T00:51:23,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683100 2024-12-07T00:51:23,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683100) roll requested 2024-12-07T00:51:23,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683108 2024-12-07T00:51:23,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683108 2024-12-07T00:51:23,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683108) roll requested 2024-12-07T00:51:23,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683115 2024-12-07T00:51:23,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683115 2024-12-07T00:51:23,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683115) roll requested 2024-12-07T00:51:23,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683123 2024-12-07T00:51:23,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683123 2024-12-07T00:51:23,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683123) roll requested 2024-12-07T00:51:23,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683131 2024-12-07T00:51:23,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683131 2024-12-07T00:51:23,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683131) roll requested 2024-12-07T00:51:23,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683139 2024-12-07T00:51:23,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683139 2024-12-07T00:51:23,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683139) roll requested 2024-12-07T00:51:23,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683146 2024-12-07T00:51:23,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683146 2024-12-07T00:51:23,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683146) roll requested 2024-12-07T00:51:23,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683152 2024-12-07T00:51:23,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683152 2024-12-07T00:51:23,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683152) roll requested 2024-12-07T00:51:23,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683157 2024-12-07T00:51:23,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683157 2024-12-07T00:51:23,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683157) roll requested 2024-12-07T00:51:23,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683162 2024-12-07T00:51:23,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683162 2024-12-07T00:51:23,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683162) roll requested 2024-12-07T00:51:23,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683169 2024-12-07T00:51:23,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683169 2024-12-07T00:51:23,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683169) roll requested 2024-12-07T00:51:23,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683176 2024-12-07T00:51:23,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683176 2024-12-07T00:51:23,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683176) roll requested 2024-12-07T00:51:23,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683185 2024-12-07T00:51:23,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683185 2024-12-07T00:51:23,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683185) roll requested 2024-12-07T00:51:23,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683195 2024-12-07T00:51:23,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683195 2024-12-07T00:51:23,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683195) roll requested 2024-12-07T00:51:23,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683201 2024-12-07T00:51:23,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683201 2024-12-07T00:51:23,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683201) roll requested 2024-12-07T00:51:23,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683208 2024-12-07T00:51:23,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683208 2024-12-07T00:51:23,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683208) roll requested 2024-12-07T00:51:23,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683213 2024-12-07T00:51:23,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683213 2024-12-07T00:51:23,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683213) roll requested 2024-12-07T00:51:23,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683218 2024-12-07T00:51:23,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683218 2024-12-07T00:51:23,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683218) roll requested 2024-12-07T00:51:23,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683224 2024-12-07T00:51:23,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683224 2024-12-07T00:51:23,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683224) roll requested 2024-12-07T00:51:23,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683230 2024-12-07T00:51:23,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683230 2024-12-07T00:51:23,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683230) roll requested 2024-12-07T00:51:23,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683237 2024-12-07T00:51:23,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683237 2024-12-07T00:51:23,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683237) roll requested 2024-12-07T00:51:23,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683247 2024-12-07T00:51:23,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683247 2024-12-07T00:51:23,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683247) roll requested 2024-12-07T00:51:23,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683256 2024-12-07T00:51:23,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683256 2024-12-07T00:51:23,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683256) roll requested 2024-12-07T00:51:23,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683263 2024-12-07T00:51:23,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683263 2024-12-07T00:51:23,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683263) roll requested 2024-12-07T00:51:23,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683270 2024-12-07T00:51:23,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683270 2024-12-07T00:51:23,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683270) roll requested 2024-12-07T00:51:23,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683278 2024-12-07T00:51:23,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683278 2024-12-07T00:51:23,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683278) roll requested 2024-12-07T00:51:23,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683286 2024-12-07T00:51:23,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683286 2024-12-07T00:51:23,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683286) roll requested 2024-12-07T00:51:23,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683294 2024-12-07T00:51:23,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683294 2024-12-07T00:51:23,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683294) roll requested 2024-12-07T00:51:23,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683301 2024-12-07T00:51:23,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683301 2024-12-07T00:51:23,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683301) roll requested 2024-12-07T00:51:23,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683309 2024-12-07T00:51:23,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683309 2024-12-07T00:51:23,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683309) roll requested 2024-12-07T00:51:23,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683317 2024-12-07T00:51:23,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683317 2024-12-07T00:51:23,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683317) roll requested 2024-12-07T00:51:23,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683386 2024-12-07T00:51:23,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683386 2024-12-07T00:51:23,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683386) roll requested 2024-12-07T00:51:23,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683392 2024-12-07T00:51:23,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683392 2024-12-07T00:51:23,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683392) roll requested 2024-12-07T00:51:23,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683399 2024-12-07T00:51:23,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683399 2024-12-07T00:51:23,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683399) roll requested 2024-12-07T00:51:23,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683407 2024-12-07T00:51:23,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683407 2024-12-07T00:51:23,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683407) roll requested 2024-12-07T00:51:23,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683414 2024-12-07T00:51:23,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683414 2024-12-07T00:51:23,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683414) roll requested 2024-12-07T00:51:23,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683425 2024-12-07T00:51:23,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683425 2024-12-07T00:51:23,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683425) roll requested 2024-12-07T00:51:23,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683439 2024-12-07T00:51:23,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683439 2024-12-07T00:51:23,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683439) roll requested 2024-12-07T00:51:23,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683444 2024-12-07T00:51:23,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683444 2024-12-07T00:51:23,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683444) roll requested 2024-12-07T00:51:23,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683452 2024-12-07T00:51:23,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683452 2024-12-07T00:51:23,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683452) roll requested 2024-12-07T00:51:23,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683460 2024-12-07T00:51:23,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683460 2024-12-07T00:51:23,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683460) roll requested 2024-12-07T00:51:23,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683465 2024-12-07T00:51:23,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683465 2024-12-07T00:51:23,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683465) roll requested 2024-12-07T00:51:23,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683472 2024-12-07T00:51:23,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683472 2024-12-07T00:51:23,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683472) roll requested 2024-12-07T00:51:23,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683479 2024-12-07T00:51:23,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683479 2024-12-07T00:51:23,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683479) roll requested 2024-12-07T00:51:23,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683488 2024-12-07T00:51:23,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683488 2024-12-07T00:51:23,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683488) roll requested 2024-12-07T00:51:23,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683503 2024-12-07T00:51:23,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683503 2024-12-07T00:51:23,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683503) roll requested 2024-12-07T00:51:23,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683509 2024-12-07T00:51:23,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683509 2024-12-07T00:51:23,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683509) roll requested 2024-12-07T00:51:23,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683516 2024-12-07T00:51:23,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683516 2024-12-07T00:51:23,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683516) roll requested 2024-12-07T00:51:23,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683524 2024-12-07T00:51:23,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683524 2024-12-07T00:51:23,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683524) roll requested 2024-12-07T00:51:23,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683532 2024-12-07T00:51:23,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683532 2024-12-07T00:51:23,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683532) roll requested 2024-12-07T00:51:23,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683539 2024-12-07T00:51:23,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683539 2024-12-07T00:51:23,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683539) roll requested 2024-12-07T00:51:23,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683544 2024-12-07T00:51:23,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683544 2024-12-07T00:51:23,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683544) roll requested 2024-12-07T00:51:23,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683550 2024-12-07T00:51:23,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683550 2024-12-07T00:51:23,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683550) roll requested 2024-12-07T00:51:23,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683556 2024-12-07T00:51:23,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683556 2024-12-07T00:51:23,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683556) roll requested 2024-12-07T00:51:23,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683562 2024-12-07T00:51:23,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683562 2024-12-07T00:51:23,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683562) roll requested 2024-12-07T00:51:23,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683568 2024-12-07T00:51:23,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683568 2024-12-07T00:51:23,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683568) roll requested 2024-12-07T00:51:23,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683577 2024-12-07T00:51:23,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683577 2024-12-07T00:51:23,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683577) roll requested 2024-12-07T00:51:23,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683583 2024-12-07T00:51:23,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683583 2024-12-07T00:51:23,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683583) roll requested 2024-12-07T00:51:23,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683588 2024-12-07T00:51:23,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683588 2024-12-07T00:51:23,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683588) roll requested 2024-12-07T00:51:23,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683598 2024-12-07T00:51:23,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683598 2024-12-07T00:51:23,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683598) roll requested 2024-12-07T00:51:23,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683607 2024-12-07T00:51:23,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683607 2024-12-07T00:51:23,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683607) roll requested 2024-12-07T00:51:23,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683612 2024-12-07T00:51:23,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683612 2024-12-07T00:51:23,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683612) roll requested 2024-12-07T00:51:23,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683618 2024-12-07T00:51:23,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683618 2024-12-07T00:51:23,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683618) roll requested 2024-12-07T00:51:23,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683625 2024-12-07T00:51:23,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683625 2024-12-07T00:51:23,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683625) roll requested 2024-12-07T00:51:23,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683631 2024-12-07T00:51:23,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683631 2024-12-07T00:51:23,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683631) roll requested 2024-12-07T00:51:23,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683636 2024-12-07T00:51:23,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683636 2024-12-07T00:51:23,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683636) roll requested 2024-12-07T00:51:23,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683641 2024-12-07T00:51:23,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683641 2024-12-07T00:51:23,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683641) roll requested 2024-12-07T00:51:23,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683648 2024-12-07T00:51:23,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683648 2024-12-07T00:51:23,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683648) roll requested 2024-12-07T00:51:23,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683655 2024-12-07T00:51:23,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683655 2024-12-07T00:51:23,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683655) roll requested 2024-12-07T00:51:23,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683663 2024-12-07T00:51:23,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683663 2024-12-07T00:51:23,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683663) roll requested 2024-12-07T00:51:23,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683671 2024-12-07T00:51:23,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683671 2024-12-07T00:51:23,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683671) roll requested 2024-12-07T00:51:23,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683680 2024-12-07T00:51:23,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683680 2024-12-07T00:51:23,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683680) roll requested 2024-12-07T00:51:23,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683689 2024-12-07T00:51:23,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683689 2024-12-07T00:51:23,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683689) roll requested 2024-12-07T00:51:23,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683695 2024-12-07T00:51:23,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683695 2024-12-07T00:51:23,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683695) roll requested 2024-12-07T00:51:23,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683702 2024-12-07T00:51:23,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683702 2024-12-07T00:51:23,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683702) roll requested 2024-12-07T00:51:23,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683710 2024-12-07T00:51:23,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683710 2024-12-07T00:51:23,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683710) roll requested 2024-12-07T00:51:23,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683719 2024-12-07T00:51:23,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683719 2024-12-07T00:51:23,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683719) roll requested 2024-12-07T00:51:23,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683728 2024-12-07T00:51:23,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683728 2024-12-07T00:51:23,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683728) roll requested 2024-12-07T00:51:23,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683735 2024-12-07T00:51:23,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683735 2024-12-07T00:51:23,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683735) roll requested 2024-12-07T00:51:23,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683743 2024-12-07T00:51:23,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683743 2024-12-07T00:51:23,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683743) roll requested 2024-12-07T00:51:23,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683750 2024-12-07T00:51:23,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683750 2024-12-07T00:51:23,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683750) roll requested 2024-12-07T00:51:23,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683758 2024-12-07T00:51:23,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683758 2024-12-07T00:51:23,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683758) roll requested 2024-12-07T00:51:23,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683767 2024-12-07T00:51:23,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683767 2024-12-07T00:51:23,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683767) roll requested 2024-12-07T00:51:23,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683775 2024-12-07T00:51:23,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683775 2024-12-07T00:51:23,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683775) roll requested 2024-12-07T00:51:23,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683784 2024-12-07T00:51:23,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683784 2024-12-07T00:51:23,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683784) roll requested 2024-12-07T00:51:23,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683793 2024-12-07T00:51:23,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683793 2024-12-07T00:51:23,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683793) roll requested 2024-12-07T00:51:23,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683802 2024-12-07T00:51:23,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683802 2024-12-07T00:51:23,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683802) roll requested 2024-12-07T00:51:23,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683814 2024-12-07T00:51:23,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683814 2024-12-07T00:51:23,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683814) roll requested 2024-12-07T00:51:23,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683823 2024-12-07T00:51:23,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683823 2024-12-07T00:51:23,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683823) roll requested 2024-12-07T00:51:23,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683832 2024-12-07T00:51:23,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683832 2024-12-07T00:51:23,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683832) roll requested 2024-12-07T00:51:23,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683843 2024-12-07T00:51:23,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683843 2024-12-07T00:51:23,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683843) roll requested 2024-12-07T00:51:23,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683851 2024-12-07T00:51:23,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683851 2024-12-07T00:51:23,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683851) roll requested 2024-12-07T00:51:23,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683858 2024-12-07T00:51:23,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683858 2024-12-07T00:51:23,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683858) roll requested 2024-12-07T00:51:23,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683865 2024-12-07T00:51:23,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683865 2024-12-07T00:51:23,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683865) roll requested 2024-12-07T00:51:23,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683871 2024-12-07T00:51:23,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683871 2024-12-07T00:51:23,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683871) roll requested 2024-12-07T00:51:23,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683878 2024-12-07T00:51:23,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683878 2024-12-07T00:51:23,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683878) roll requested 2024-12-07T00:51:23,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683886 2024-12-07T00:51:23,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683886 2024-12-07T00:51:23,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683886) roll requested 2024-12-07T00:51:23,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683895 2024-12-07T00:51:23,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683895 2024-12-07T00:51:23,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683895) roll requested 2024-12-07T00:51:23,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683903 2024-12-07T00:51:23,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683903 2024-12-07T00:51:23,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683903) roll requested 2024-12-07T00:51:23,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683913 2024-12-07T00:51:23,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683913 2024-12-07T00:51:23,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683913) roll requested 2024-12-07T00:51:23,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683922 2024-12-07T00:51:23,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683922 2024-12-07T00:51:23,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683922) roll requested 2024-12-07T00:51:23,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683930 2024-12-07T00:51:23,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683930 2024-12-07T00:51:23,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683930) roll requested 2024-12-07T00:51:23,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683939 2024-12-07T00:51:23,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683939 2024-12-07T00:51:23,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683939) roll requested 2024-12-07T00:51:23,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683947 2024-12-07T00:51:23,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683947 2024-12-07T00:51:23,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683947) roll requested 2024-12-07T00:51:23,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683954 2024-12-07T00:51:23,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683954 2024-12-07T00:51:23,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683954) roll requested 2024-12-07T00:51:23,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683964 2024-12-07T00:51:23,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683964 2024-12-07T00:51:23,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683964) roll requested 2024-12-07T00:51:23,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683973 2024-12-07T00:51:23,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683973 2024-12-07T00:51:23,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683973) roll requested 2024-12-07T00:51:23,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683982 2024-12-07T00:51:23,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683982 2024-12-07T00:51:23,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683982) roll requested 2024-12-07T00:51:23,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683991 2024-12-07T00:51:23,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:23,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683991 2024-12-07T00:51:23,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:23,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:23,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683991) roll requested 2024-12-07T00:51:23,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532683997 2024-12-07T00:51:24,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683997 2024-12-07T00:51:24,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532683997) roll requested 2024-12-07T00:51:24,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684004 2024-12-07T00:51:24,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532683997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684004 2024-12-07T00:51:24,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684004) roll requested 2024-12-07T00:51:24,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684012 2024-12-07T00:51:24,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684012 2024-12-07T00:51:24,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684012) roll requested 2024-12-07T00:51:24,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684020 2024-12-07T00:51:24,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684020 2024-12-07T00:51:24,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684020) roll requested 2024-12-07T00:51:24,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684028 2024-12-07T00:51:24,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684028 2024-12-07T00:51:24,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684028) roll requested 2024-12-07T00:51:24,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684035 2024-12-07T00:51:24,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684035 2024-12-07T00:51:24,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684035) roll requested 2024-12-07T00:51:24,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684045 2024-12-07T00:51:24,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684045 2024-12-07T00:51:24,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684045) roll requested 2024-12-07T00:51:24,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684052 2024-12-07T00:51:24,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684052 2024-12-07T00:51:24,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684052) roll requested 2024-12-07T00:51:24,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684059 2024-12-07T00:51:24,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684059 2024-12-07T00:51:24,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684059) roll requested 2024-12-07T00:51:24,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684067 2024-12-07T00:51:24,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684067 2024-12-07T00:51:24,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684067) roll requested 2024-12-07T00:51:24,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684075 2024-12-07T00:51:24,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684075 2024-12-07T00:51:24,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684075) roll requested 2024-12-07T00:51:24,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684083 2024-12-07T00:51:24,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684083 2024-12-07T00:51:24,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684083) roll requested 2024-12-07T00:51:24,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684092 2024-12-07T00:51:24,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684092 2024-12-07T00:51:24,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684092) roll requested 2024-12-07T00:51:24,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684099 2024-12-07T00:51:24,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684099 2024-12-07T00:51:24,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684099) roll requested 2024-12-07T00:51:24,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684104 2024-12-07T00:51:24,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684104 2024-12-07T00:51:24,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684104) roll requested 2024-12-07T00:51:24,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684109 2024-12-07T00:51:24,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684109 2024-12-07T00:51:24,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684109) roll requested 2024-12-07T00:51:24,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684114 2024-12-07T00:51:24,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684114 2024-12-07T00:51:24,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684114) roll requested 2024-12-07T00:51:24,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684120 2024-12-07T00:51:24,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684120 2024-12-07T00:51:24,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684120) roll requested 2024-12-07T00:51:24,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684127 2024-12-07T00:51:24,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684127 2024-12-07T00:51:24,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684127) roll requested 2024-12-07T00:51:24,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684133 2024-12-07T00:51:24,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684133 2024-12-07T00:51:24,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684133) roll requested 2024-12-07T00:51:24,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684139 2024-12-07T00:51:24,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684139 2024-12-07T00:51:24,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684139) roll requested 2024-12-07T00:51:24,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684145 2024-12-07T00:51:24,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684145 2024-12-07T00:51:24,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684145) roll requested 2024-12-07T00:51:24,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684154 2024-12-07T00:51:24,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684154 2024-12-07T00:51:24,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684154) roll requested 2024-12-07T00:51:24,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684161 2024-12-07T00:51:24,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684161 2024-12-07T00:51:24,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684161) roll requested 2024-12-07T00:51:24,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684167 2024-12-07T00:51:24,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684167 2024-12-07T00:51:24,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684167) roll requested 2024-12-07T00:51:24,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684173 2024-12-07T00:51:24,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684173 2024-12-07T00:51:24,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684173) roll requested 2024-12-07T00:51:24,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684180 2024-12-07T00:51:24,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684180 2024-12-07T00:51:24,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684180) roll requested 2024-12-07T00:51:24,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684188 2024-12-07T00:51:24,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684188 2024-12-07T00:51:24,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684188) roll requested 2024-12-07T00:51:24,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684194 2024-12-07T00:51:24,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684194 2024-12-07T00:51:24,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684194) roll requested 2024-12-07T00:51:24,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684202 2024-12-07T00:51:24,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684202 2024-12-07T00:51:24,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684202) roll requested 2024-12-07T00:51:24,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684209 2024-12-07T00:51:24,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684209 2024-12-07T00:51:24,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684209) roll requested 2024-12-07T00:51:24,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684216 2024-12-07T00:51:24,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684216 2024-12-07T00:51:24,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684216) roll requested 2024-12-07T00:51:24,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684224 2024-12-07T00:51:24,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684224 2024-12-07T00:51:24,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684224) roll requested 2024-12-07T00:51:24,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684231 2024-12-07T00:51:24,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684231 2024-12-07T00:51:24,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684231) roll requested 2024-12-07T00:51:24,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684238 2024-12-07T00:51:24,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684238 2024-12-07T00:51:24,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684238) roll requested 2024-12-07T00:51:24,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684244 2024-12-07T00:51:24,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684244 2024-12-07T00:51:24,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684244) roll requested 2024-12-07T00:51:24,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684250 2024-12-07T00:51:24,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684250 2024-12-07T00:51:24,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684250) roll requested 2024-12-07T00:51:24,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684256 2024-12-07T00:51:24,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684256 2024-12-07T00:51:24,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684256) roll requested 2024-12-07T00:51:24,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684267 2024-12-07T00:51:24,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684267 2024-12-07T00:51:24,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684267) roll requested 2024-12-07T00:51:24,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684273 2024-12-07T00:51:24,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684273 2024-12-07T00:51:24,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684273) roll requested 2024-12-07T00:51:24,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684278 2024-12-07T00:51:24,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684278 2024-12-07T00:51:24,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684278) roll requested 2024-12-07T00:51:24,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684283 2024-12-07T00:51:24,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684283 2024-12-07T00:51:24,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684283) roll requested 2024-12-07T00:51:24,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684289 2024-12-07T00:51:24,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684289 2024-12-07T00:51:24,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684289) roll requested 2024-12-07T00:51:24,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684295 2024-12-07T00:51:24,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684295 2024-12-07T00:51:24,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684295) roll requested 2024-12-07T00:51:24,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684300 2024-12-07T00:51:24,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684300 2024-12-07T00:51:24,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684300) roll requested 2024-12-07T00:51:24,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684305 2024-12-07T00:51:24,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684305 2024-12-07T00:51:24,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684305) roll requested 2024-12-07T00:51:24,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684310 2024-12-07T00:51:24,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684310 2024-12-07T00:51:24,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684310) roll requested 2024-12-07T00:51:24,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684315 2024-12-07T00:51:24,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684315 2024-12-07T00:51:24,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684315) roll requested 2024-12-07T00:51:24,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684320 2024-12-07T00:51:24,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684320 2024-12-07T00:51:24,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684320) roll requested 2024-12-07T00:51:24,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684325 2024-12-07T00:51:24,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684325 2024-12-07T00:51:24,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684325) roll requested 2024-12-07T00:51:24,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684330 2024-12-07T00:51:24,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684330 2024-12-07T00:51:24,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684330) roll requested 2024-12-07T00:51:24,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684336 2024-12-07T00:51:24,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684336 2024-12-07T00:51:24,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684336) roll requested 2024-12-07T00:51:24,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684342 2024-12-07T00:51:24,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684342 2024-12-07T00:51:24,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684342) roll requested 2024-12-07T00:51:24,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684349 2024-12-07T00:51:24,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684349 2024-12-07T00:51:24,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684349) roll requested 2024-12-07T00:51:24,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684354 2024-12-07T00:51:24,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684354 2024-12-07T00:51:24,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684354) roll requested 2024-12-07T00:51:24,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684360 2024-12-07T00:51:24,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684360 2024-12-07T00:51:24,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684360) roll requested 2024-12-07T00:51:24,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684366 2024-12-07T00:51:24,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684366 2024-12-07T00:51:24,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684366) roll requested 2024-12-07T00:51:24,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684371 2024-12-07T00:51:24,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684371 2024-12-07T00:51:24,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684371) roll requested 2024-12-07T00:51:24,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684376 2024-12-07T00:51:24,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684376 2024-12-07T00:51:24,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684376) roll requested 2024-12-07T00:51:24,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684381 2024-12-07T00:51:24,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684381 2024-12-07T00:51:24,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684381) roll requested 2024-12-07T00:51:24,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684394 2024-12-07T00:51:24,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684394 2024-12-07T00:51:24,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684394) roll requested 2024-12-07T00:51:24,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684407 2024-12-07T00:51:24,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684407 2024-12-07T00:51:24,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684407) roll requested 2024-12-07T00:51:24,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684413 2024-12-07T00:51:24,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684413 2024-12-07T00:51:24,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684413) roll requested 2024-12-07T00:51:24,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684430 2024-12-07T00:51:24,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684430 2024-12-07T00:51:24,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684430) roll requested 2024-12-07T00:51:24,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684456 2024-12-07T00:51:24,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684456 2024-12-07T00:51:24,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684456) roll requested 2024-12-07T00:51:24,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684463 2024-12-07T00:51:24,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684463 2024-12-07T00:51:24,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684463) roll requested 2024-12-07T00:51:24,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684471 2024-12-07T00:51:24,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684471 2024-12-07T00:51:24,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684471) roll requested 2024-12-07T00:51:24,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684478 2024-12-07T00:51:24,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684478 2024-12-07T00:51:24,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684478) roll requested 2024-12-07T00:51:24,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684488 2024-12-07T00:51:24,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684488 2024-12-07T00:51:24,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684488) roll requested 2024-12-07T00:51:24,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684500 2024-12-07T00:51:24,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684500 2024-12-07T00:51:24,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684500) roll requested 2024-12-07T00:51:24,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684506 2024-12-07T00:51:24,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684506 2024-12-07T00:51:24,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684506) roll requested 2024-12-07T00:51:24,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684512 2024-12-07T00:51:24,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684512 2024-12-07T00:51:24,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684512) roll requested 2024-12-07T00:51:24,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684518 2024-12-07T00:51:24,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684518 2024-12-07T00:51:24,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684518) roll requested 2024-12-07T00:51:24,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684525 2024-12-07T00:51:24,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684525 2024-12-07T00:51:24,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684525) roll requested 2024-12-07T00:51:24,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684531 2024-12-07T00:51:24,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684531 2024-12-07T00:51:24,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684531) roll requested 2024-12-07T00:51:24,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684536 2024-12-07T00:51:24,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684536 2024-12-07T00:51:24,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684536) roll requested 2024-12-07T00:51:24,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684541 2024-12-07T00:51:24,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684541 2024-12-07T00:51:24,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684541) roll requested 2024-12-07T00:51:24,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684546 2024-12-07T00:51:24,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684546 2024-12-07T00:51:24,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684546) roll requested 2024-12-07T00:51:24,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684552 2024-12-07T00:51:24,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684552 2024-12-07T00:51:24,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684552) roll requested 2024-12-07T00:51:24,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684560 2024-12-07T00:51:24,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684560 2024-12-07T00:51:24,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684560) roll requested 2024-12-07T00:51:24,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684567 2024-12-07T00:51:24,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684567 2024-12-07T00:51:24,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684567) roll requested 2024-12-07T00:51:24,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684574 2024-12-07T00:51:24,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684574 2024-12-07T00:51:24,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684574) roll requested 2024-12-07T00:51:24,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684582 2024-12-07T00:51:24,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684582 2024-12-07T00:51:24,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684582) roll requested 2024-12-07T00:51:24,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684590 2024-12-07T00:51:24,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684590 2024-12-07T00:51:24,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684590) roll requested 2024-12-07T00:51:24,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684597 2024-12-07T00:51:24,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684597 2024-12-07T00:51:24,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684597) roll requested 2024-12-07T00:51:24,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684604 2024-12-07T00:51:24,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684604 2024-12-07T00:51:24,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684604) roll requested 2024-12-07T00:51:24,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684613 2024-12-07T00:51:24,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684613 2024-12-07T00:51:24,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684613) roll requested 2024-12-07T00:51:24,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684620 2024-12-07T00:51:24,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684620 2024-12-07T00:51:24,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684620) roll requested 2024-12-07T00:51:24,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684629 2024-12-07T00:51:24,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684629 2024-12-07T00:51:24,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684629) roll requested 2024-12-07T00:51:24,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684638 2024-12-07T00:51:24,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684638 2024-12-07T00:51:24,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684638) roll requested 2024-12-07T00:51:24,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684647 2024-12-07T00:51:24,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684647 2024-12-07T00:51:24,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684647) roll requested 2024-12-07T00:51:24,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684654 2024-12-07T00:51:24,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684654 2024-12-07T00:51:24,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684654) roll requested 2024-12-07T00:51:24,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684661 2024-12-07T00:51:24,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684661 2024-12-07T00:51:24,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684661) roll requested 2024-12-07T00:51:24,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684668 2024-12-07T00:51:24,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684668 2024-12-07T00:51:24,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684668) roll requested 2024-12-07T00:51:24,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684678 2024-12-07T00:51:24,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684678 2024-12-07T00:51:24,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684678) roll requested 2024-12-07T00:51:24,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684686 2024-12-07T00:51:24,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684686 2024-12-07T00:51:24,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684686) roll requested 2024-12-07T00:51:24,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684693 2024-12-07T00:51:24,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684693 2024-12-07T00:51:24,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684693) roll requested 2024-12-07T00:51:24,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684698 2024-12-07T00:51:24,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684698 2024-12-07T00:51:24,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684698) roll requested 2024-12-07T00:51:24,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684703 2024-12-07T00:51:24,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684703 2024-12-07T00:51:24,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684703) roll requested 2024-12-07T00:51:24,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684708 2024-12-07T00:51:24,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684708 2024-12-07T00:51:24,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684708) roll requested 2024-12-07T00:51:24,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684713 2024-12-07T00:51:24,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684713 2024-12-07T00:51:24,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684713) roll requested 2024-12-07T00:51:24,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684720 2024-12-07T00:51:24,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684720 2024-12-07T00:51:24,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684720) roll requested 2024-12-07T00:51:24,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684726 2024-12-07T00:51:24,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684726 2024-12-07T00:51:24,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684726) roll requested 2024-12-07T00:51:24,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684734 2024-12-07T00:51:24,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684734 2024-12-07T00:51:24,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684734) roll requested 2024-12-07T00:51:24,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684742 2024-12-07T00:51:24,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684742 2024-12-07T00:51:24,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684742) roll requested 2024-12-07T00:51:24,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684750 2024-12-07T00:51:24,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684750 2024-12-07T00:51:24,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684750) roll requested 2024-12-07T00:51:24,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684757 2024-12-07T00:51:24,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684757 2024-12-07T00:51:24,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684757) roll requested 2024-12-07T00:51:24,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684766 2024-12-07T00:51:24,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684766 2024-12-07T00:51:24,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684766) roll requested 2024-12-07T00:51:24,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684774 2024-12-07T00:51:24,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684774 2024-12-07T00:51:24,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684774) roll requested 2024-12-07T00:51:24,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684784 2024-12-07T00:51:24,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684784 2024-12-07T00:51:24,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684784) roll requested 2024-12-07T00:51:24,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684790 2024-12-07T00:51:24,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684790 2024-12-07T00:51:24,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684790) roll requested 2024-12-07T00:51:24,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684795 2024-12-07T00:51:24,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684795 2024-12-07T00:51:24,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684795) roll requested 2024-12-07T00:51:24,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684800 2024-12-07T00:51:24,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684800 2024-12-07T00:51:24,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684800) roll requested 2024-12-07T00:51:24,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684805 2024-12-07T00:51:24,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684805 2024-12-07T00:51:24,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684805) roll requested 2024-12-07T00:51:24,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684811 2024-12-07T00:51:24,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684811 2024-12-07T00:51:24,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684811) roll requested 2024-12-07T00:51:24,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684822 2024-12-07T00:51:24,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684822 2024-12-07T00:51:24,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684822) roll requested 2024-12-07T00:51:24,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684829 2024-12-07T00:51:24,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684829 2024-12-07T00:51:24,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684829) roll requested 2024-12-07T00:51:24,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684838 2024-12-07T00:51:24,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684838 2024-12-07T00:51:24,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684838) roll requested 2024-12-07T00:51:24,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684844 2024-12-07T00:51:24,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684844 2024-12-07T00:51:24,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684844) roll requested 2024-12-07T00:51:24,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684850 2024-12-07T00:51:24,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684850 2024-12-07T00:51:24,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684850) roll requested 2024-12-07T00:51:24,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684855 2024-12-07T00:51:24,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684855 2024-12-07T00:51:24,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684855) roll requested 2024-12-07T00:51:24,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684861 2024-12-07T00:51:24,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684861 2024-12-07T00:51:24,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684861) roll requested 2024-12-07T00:51:24,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684867 2024-12-07T00:51:24,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684867 2024-12-07T00:51:24,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684867) roll requested 2024-12-07T00:51:24,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684872 2024-12-07T00:51:24,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684872 2024-12-07T00:51:24,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684872) roll requested 2024-12-07T00:51:24,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684878 2024-12-07T00:51:24,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684878 2024-12-07T00:51:24,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684878) roll requested 2024-12-07T00:51:24,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684884 2024-12-07T00:51:24,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684884 2024-12-07T00:51:24,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684884) roll requested 2024-12-07T00:51:24,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684889 2024-12-07T00:51:24,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684889 2024-12-07T00:51:24,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684889) roll requested 2024-12-07T00:51:24,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684894 2024-12-07T00:51:24,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684894 2024-12-07T00:51:24,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684894) roll requested 2024-12-07T00:51:24,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684900 2024-12-07T00:51:24,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684900 2024-12-07T00:51:24,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684900) roll requested 2024-12-07T00:51:24,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684906 2024-12-07T00:51:24,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684906 2024-12-07T00:51:24,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684906) roll requested 2024-12-07T00:51:24,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684912 2024-12-07T00:51:24,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684912 2024-12-07T00:51:24,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684912) roll requested 2024-12-07T00:51:24,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684917 2024-12-07T00:51:24,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684917 2024-12-07T00:51:24,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684917) roll requested 2024-12-07T00:51:24,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684922 2024-12-07T00:51:24,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684922 2024-12-07T00:51:24,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684922) roll requested 2024-12-07T00:51:24,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684927 2024-12-07T00:51:24,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684927 2024-12-07T00:51:24,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684927) roll requested 2024-12-07T00:51:24,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684933 2024-12-07T00:51:24,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684933 2024-12-07T00:51:24,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684933) roll requested 2024-12-07T00:51:24,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684938 2024-12-07T00:51:24,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684938 2024-12-07T00:51:24,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684938) roll requested 2024-12-07T00:51:24,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684943 2024-12-07T00:51:24,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684943 2024-12-07T00:51:24,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684943) roll requested 2024-12-07T00:51:24,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684949 2024-12-07T00:51:24,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684949 2024-12-07T00:51:24,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684949) roll requested 2024-12-07T00:51:24,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684955 2024-12-07T00:51:24,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684955 2024-12-07T00:51:24,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684955) roll requested 2024-12-07T00:51:24,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684961 2024-12-07T00:51:24,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684961 2024-12-07T00:51:24,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684961) roll requested 2024-12-07T00:51:24,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684966 2024-12-07T00:51:24,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684966 2024-12-07T00:51:24,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684966) roll requested 2024-12-07T00:51:24,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684971 2024-12-07T00:51:24,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684971 2024-12-07T00:51:24,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684971) roll requested 2024-12-07T00:51:24,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684977 2024-12-07T00:51:24,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684977 2024-12-07T00:51:24,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684977) roll requested 2024-12-07T00:51:24,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684984 2024-12-07T00:51:24,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684984 2024-12-07T00:51:24,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:24,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:24,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684984) roll requested 2024-12-07T00:51:24,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532684992 2024-12-07T00:51:24,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:24,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684992 2024-12-07T00:51:25,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532684992) roll requested 2024-12-07T00:51:25,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685000 2024-12-07T00:51:25,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532684992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685000 2024-12-07T00:51:25,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685000) roll requested 2024-12-07T00:51:25,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685008 2024-12-07T00:51:25,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685008 2024-12-07T00:51:25,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685008) roll requested 2024-12-07T00:51:25,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685017 2024-12-07T00:51:25,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685017 2024-12-07T00:51:25,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685017) roll requested 2024-12-07T00:51:25,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685024 2024-12-07T00:51:25,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685024 2024-12-07T00:51:25,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685024) roll requested 2024-12-07T00:51:25,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685031 2024-12-07T00:51:25,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685031 2024-12-07T00:51:25,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685031) roll requested 2024-12-07T00:51:25,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685040 2024-12-07T00:51:25,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685040 2024-12-07T00:51:25,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685040) roll requested 2024-12-07T00:51:25,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685048 2024-12-07T00:51:25,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685048 2024-12-07T00:51:25,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685048) roll requested 2024-12-07T00:51:25,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685056 2024-12-07T00:51:25,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685056 2024-12-07T00:51:25,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685056) roll requested 2024-12-07T00:51:25,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685064 2024-12-07T00:51:25,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685064 2024-12-07T00:51:25,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685064) roll requested 2024-12-07T00:51:25,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685072 2024-12-07T00:51:25,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685072 2024-12-07T00:51:25,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685072) roll requested 2024-12-07T00:51:25,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685082 2024-12-07T00:51:25,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685082 2024-12-07T00:51:25,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685082) roll requested 2024-12-07T00:51:25,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685090 2024-12-07T00:51:25,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685090 2024-12-07T00:51:25,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685090) roll requested 2024-12-07T00:51:25,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685097 2024-12-07T00:51:25,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685097 2024-12-07T00:51:25,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685097) roll requested 2024-12-07T00:51:25,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685105 2024-12-07T00:51:25,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685105 2024-12-07T00:51:25,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685105) roll requested 2024-12-07T00:51:25,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685114 2024-12-07T00:51:25,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685114 2024-12-07T00:51:25,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685114) roll requested 2024-12-07T00:51:25,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685124 2024-12-07T00:51:25,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685124 2024-12-07T00:51:25,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685124) roll requested 2024-12-07T00:51:25,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685133 2024-12-07T00:51:25,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685133 2024-12-07T00:51:25,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685133) roll requested 2024-12-07T00:51:25,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685142 2024-12-07T00:51:25,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685142 2024-12-07T00:51:25,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685142) roll requested 2024-12-07T00:51:25,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685154 2024-12-07T00:51:25,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685154 2024-12-07T00:51:25,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685154) roll requested 2024-12-07T00:51:25,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685161 2024-12-07T00:51:25,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685161 2024-12-07T00:51:25,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685161) roll requested 2024-12-07T00:51:25,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685168 2024-12-07T00:51:25,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685168 2024-12-07T00:51:25,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685168) roll requested 2024-12-07T00:51:25,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685176 2024-12-07T00:51:25,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685176 2024-12-07T00:51:25,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685176) roll requested 2024-12-07T00:51:25,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685183 2024-12-07T00:51:25,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685183 2024-12-07T00:51:25,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685183) roll requested 2024-12-07T00:51:25,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685191 2024-12-07T00:51:25,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685191 2024-12-07T00:51:25,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685191) roll requested 2024-12-07T00:51:25,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685201 2024-12-07T00:51:25,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685201 2024-12-07T00:51:25,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685201) roll requested 2024-12-07T00:51:25,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685210 2024-12-07T00:51:25,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685210 2024-12-07T00:51:25,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685210) roll requested 2024-12-07T00:51:25,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685217 2024-12-07T00:51:25,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685217 2024-12-07T00:51:25,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685217) roll requested 2024-12-07T00:51:25,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685226 2024-12-07T00:51:25,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685226 2024-12-07T00:51:25,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685226) roll requested 2024-12-07T00:51:25,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685236 2024-12-07T00:51:25,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685236 2024-12-07T00:51:25,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685236) roll requested 2024-12-07T00:51:25,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685241 2024-12-07T00:51:25,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685241 2024-12-07T00:51:25,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685241) roll requested 2024-12-07T00:51:25,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685246 2024-12-07T00:51:25,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685246 2024-12-07T00:51:25,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685246) roll requested 2024-12-07T00:51:25,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685252 2024-12-07T00:51:25,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685252 2024-12-07T00:51:25,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685252) roll requested 2024-12-07T00:51:25,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685258 2024-12-07T00:51:25,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685258 2024-12-07T00:51:25,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685258) roll requested 2024-12-07T00:51:25,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685265 2024-12-07T00:51:25,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685265 2024-12-07T00:51:25,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685265) roll requested 2024-12-07T00:51:25,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685272 2024-12-07T00:51:25,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685272 2024-12-07T00:51:25,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685272) roll requested 2024-12-07T00:51:25,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685280 2024-12-07T00:51:25,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685280 2024-12-07T00:51:25,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685280) roll requested 2024-12-07T00:51:25,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685289 2024-12-07T00:51:25,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685289 2024-12-07T00:51:25,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685289) roll requested 2024-12-07T00:51:25,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685296 2024-12-07T00:51:25,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685296 2024-12-07T00:51:25,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685296) roll requested 2024-12-07T00:51:25,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685302 2024-12-07T00:51:25,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685302 2024-12-07T00:51:25,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685302) roll requested 2024-12-07T00:51:25,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685309 2024-12-07T00:51:25,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685309 2024-12-07T00:51:25,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685309) roll requested 2024-12-07T00:51:25,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685317 2024-12-07T00:51:25,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685317 2024-12-07T00:51:25,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685317) roll requested 2024-12-07T00:51:25,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685326 2024-12-07T00:51:25,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685326 2024-12-07T00:51:25,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685326) roll requested 2024-12-07T00:51:25,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685335 2024-12-07T00:51:25,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685335 2024-12-07T00:51:25,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685335) roll requested 2024-12-07T00:51:25,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685345 2024-12-07T00:51:25,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685345 2024-12-07T00:51:25,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685345) roll requested 2024-12-07T00:51:25,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685353 2024-12-07T00:51:25,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685353 2024-12-07T00:51:25,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685353) roll requested 2024-12-07T00:51:25,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685361 2024-12-07T00:51:25,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685361 2024-12-07T00:51:25,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685361) roll requested 2024-12-07T00:51:25,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685371 2024-12-07T00:51:25,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685371 2024-12-07T00:51:25,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685371) roll requested 2024-12-07T00:51:25,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685380 2024-12-07T00:51:25,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685380 2024-12-07T00:51:25,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685380) roll requested 2024-12-07T00:51:25,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685387 2024-12-07T00:51:25,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685387 2024-12-07T00:51:25,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685387) roll requested 2024-12-07T00:51:25,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685392 2024-12-07T00:51:25,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685392 2024-12-07T00:51:25,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685392) roll requested 2024-12-07T00:51:25,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685398 2024-12-07T00:51:25,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685398 2024-12-07T00:51:25,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685398) roll requested 2024-12-07T00:51:25,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685406 2024-12-07T00:51:25,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685406 2024-12-07T00:51:25,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685406) roll requested 2024-12-07T00:51:25,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685414 2024-12-07T00:51:25,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685414 2024-12-07T00:51:25,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685414) roll requested 2024-12-07T00:51:25,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685423 2024-12-07T00:51:25,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685423 2024-12-07T00:51:25,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685423) roll requested 2024-12-07T00:51:25,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685431 2024-12-07T00:51:25,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685431 2024-12-07T00:51:25,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685431) roll requested 2024-12-07T00:51:25,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685438 2024-12-07T00:51:25,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685438 2024-12-07T00:51:25,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685438) roll requested 2024-12-07T00:51:25,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685445 2024-12-07T00:51:25,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685445 2024-12-07T00:51:25,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685445) roll requested 2024-12-07T00:51:25,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685452 2024-12-07T00:51:25,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685452 2024-12-07T00:51:25,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685452) roll requested 2024-12-07T00:51:25,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685461 2024-12-07T00:51:25,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685461 2024-12-07T00:51:25,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685461) roll requested 2024-12-07T00:51:25,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685469 2024-12-07T00:51:25,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685469 2024-12-07T00:51:25,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685469) roll requested 2024-12-07T00:51:25,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685475 2024-12-07T00:51:25,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685475 2024-12-07T00:51:25,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685475) roll requested 2024-12-07T00:51:25,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685483 2024-12-07T00:51:25,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685483 2024-12-07T00:51:25,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685483) roll requested 2024-12-07T00:51:25,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685490 2024-12-07T00:51:25,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685490 2024-12-07T00:51:25,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685490) roll requested 2024-12-07T00:51:25,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685499 2024-12-07T00:51:25,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685499 2024-12-07T00:51:25,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685499) roll requested 2024-12-07T00:51:25,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685506 2024-12-07T00:51:25,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685506 2024-12-07T00:51:25,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685506) roll requested 2024-12-07T00:51:25,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685515 2024-12-07T00:51:25,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685515 2024-12-07T00:51:25,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685515) roll requested 2024-12-07T00:51:25,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685523 2024-12-07T00:51:25,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685523 2024-12-07T00:51:25,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685523) roll requested 2024-12-07T00:51:25,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685532 2024-12-07T00:51:25,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685532 2024-12-07T00:51:25,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685532) roll requested 2024-12-07T00:51:25,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685541 2024-12-07T00:51:25,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685541 2024-12-07T00:51:25,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685541) roll requested 2024-12-07T00:51:25,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685551 2024-12-07T00:51:25,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685551 2024-12-07T00:51:25,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685551) roll requested 2024-12-07T00:51:25,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685561 2024-12-07T00:51:25,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685561 2024-12-07T00:51:25,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685561) roll requested 2024-12-07T00:51:25,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685569 2024-12-07T00:51:25,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685569 2024-12-07T00:51:25,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685569) roll requested 2024-12-07T00:51:25,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685576 2024-12-07T00:51:25,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685576 2024-12-07T00:51:25,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685576) roll requested 2024-12-07T00:51:25,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685583 2024-12-07T00:51:25,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685583 2024-12-07T00:51:25,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685583) roll requested 2024-12-07T00:51:25,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685590 2024-12-07T00:51:25,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685590 2024-12-07T00:51:25,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685590) roll requested 2024-12-07T00:51:25,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685598 2024-12-07T00:51:25,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685598 2024-12-07T00:51:25,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685598) roll requested 2024-12-07T00:51:25,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685607 2024-12-07T00:51:25,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685607 2024-12-07T00:51:25,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685607) roll requested 2024-12-07T00:51:25,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685616 2024-12-07T00:51:25,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685616 2024-12-07T00:51:25,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685616) roll requested 2024-12-07T00:51:25,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685623 2024-12-07T00:51:25,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685623 2024-12-07T00:51:25,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685623) roll requested 2024-12-07T00:51:25,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685630 2024-12-07T00:51:25,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685630 2024-12-07T00:51:25,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685630) roll requested 2024-12-07T00:51:25,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685638 2024-12-07T00:51:25,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685638 2024-12-07T00:51:25,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685638) roll requested 2024-12-07T00:51:25,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685644 2024-12-07T00:51:25,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685644 2024-12-07T00:51:25,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685644) roll requested 2024-12-07T00:51:25,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685649 2024-12-07T00:51:25,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685649 2024-12-07T00:51:25,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685649) roll requested 2024-12-07T00:51:25,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685656 2024-12-07T00:51:25,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685656 2024-12-07T00:51:25,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685656) roll requested 2024-12-07T00:51:25,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685664 2024-12-07T00:51:25,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685664 2024-12-07T00:51:25,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685664) roll requested 2024-12-07T00:51:25,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685669 2024-12-07T00:51:25,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685669 2024-12-07T00:51:25,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685669) roll requested 2024-12-07T00:51:25,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685674 2024-12-07T00:51:25,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685674 2024-12-07T00:51:25,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685674) roll requested 2024-12-07T00:51:25,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685681 2024-12-07T00:51:25,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685681 2024-12-07T00:51:25,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685681) roll requested 2024-12-07T00:51:25,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685688 2024-12-07T00:51:25,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685688 2024-12-07T00:51:25,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685688) roll requested 2024-12-07T00:51:25,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685697 2024-12-07T00:51:25,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685697 2024-12-07T00:51:25,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685697) roll requested 2024-12-07T00:51:25,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685707 2024-12-07T00:51:25,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685707 2024-12-07T00:51:25,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685707) roll requested 2024-12-07T00:51:25,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685717 2024-12-07T00:51:25,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685717 2024-12-07T00:51:25,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685717) roll requested 2024-12-07T00:51:25,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685726 2024-12-07T00:51:25,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685726 2024-12-07T00:51:25,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685726) roll requested 2024-12-07T00:51:25,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685736 2024-12-07T00:51:25,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685736 2024-12-07T00:51:25,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685736) roll requested 2024-12-07T00:51:25,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685746 2024-12-07T00:51:25,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685746 2024-12-07T00:51:25,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685746) roll requested 2024-12-07T00:51:25,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685752 2024-12-07T00:51:25,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685752 2024-12-07T00:51:25,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685752) roll requested 2024-12-07T00:51:25,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685758 2024-12-07T00:51:25,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685758 2024-12-07T00:51:25,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685758) roll requested 2024-12-07T00:51:25,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685763 2024-12-07T00:51:25,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685763 2024-12-07T00:51:25,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685763) roll requested 2024-12-07T00:51:25,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685768 2024-12-07T00:51:25,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685768 2024-12-07T00:51:25,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685768) roll requested 2024-12-07T00:51:25,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685773 2024-12-07T00:51:25,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685773 2024-12-07T00:51:25,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685773) roll requested 2024-12-07T00:51:25,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685778 2024-12-07T00:51:25,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685778 2024-12-07T00:51:25,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685778) roll requested 2024-12-07T00:51:25,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685784 2024-12-07T00:51:25,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685784 2024-12-07T00:51:25,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685784) roll requested 2024-12-07T00:51:25,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685789 2024-12-07T00:51:25,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685789 2024-12-07T00:51:25,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685789) roll requested 2024-12-07T00:51:25,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685794 2024-12-07T00:51:25,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685794 2024-12-07T00:51:25,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685794) roll requested 2024-12-07T00:51:25,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685800 2024-12-07T00:51:25,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685800 2024-12-07T00:51:25,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685800) roll requested 2024-12-07T00:51:25,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685805 2024-12-07T00:51:25,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685805 2024-12-07T00:51:25,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685805) roll requested 2024-12-07T00:51:25,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685811 2024-12-07T00:51:25,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685811 2024-12-07T00:51:25,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685811) roll requested 2024-12-07T00:51:25,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685817 2024-12-07T00:51:25,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685817 2024-12-07T00:51:25,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685817) roll requested 2024-12-07T00:51:25,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685825 2024-12-07T00:51:25,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685825 2024-12-07T00:51:25,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685825) roll requested 2024-12-07T00:51:25,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685831 2024-12-07T00:51:25,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685831 2024-12-07T00:51:25,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685831) roll requested 2024-12-07T00:51:25,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685836 2024-12-07T00:51:25,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685836 2024-12-07T00:51:25,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685836) roll requested 2024-12-07T00:51:25,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685842 2024-12-07T00:51:25,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685842 2024-12-07T00:51:25,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685842) roll requested 2024-12-07T00:51:25,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685847 2024-12-07T00:51:25,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685847 2024-12-07T00:51:25,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685847) roll requested 2024-12-07T00:51:25,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685853 2024-12-07T00:51:25,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685853 2024-12-07T00:51:25,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685853) roll requested 2024-12-07T00:51:25,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685859 2024-12-07T00:51:25,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685859 2024-12-07T00:51:25,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685859) roll requested 2024-12-07T00:51:25,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685864 2024-12-07T00:51:25,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685864 2024-12-07T00:51:25,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685864) roll requested 2024-12-07T00:51:25,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685869 2024-12-07T00:51:25,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685869 2024-12-07T00:51:25,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685869) roll requested 2024-12-07T00:51:25,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685875 2024-12-07T00:51:25,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685875 2024-12-07T00:51:25,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685875) roll requested 2024-12-07T00:51:25,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685881 2024-12-07T00:51:25,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685881 2024-12-07T00:51:25,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685881) roll requested 2024-12-07T00:51:25,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685887 2024-12-07T00:51:25,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685887 2024-12-07T00:51:25,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685887) roll requested 2024-12-07T00:51:25,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685893 2024-12-07T00:51:25,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685893 2024-12-07T00:51:25,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685893) roll requested 2024-12-07T00:51:25,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685899 2024-12-07T00:51:25,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685899 2024-12-07T00:51:25,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685899) roll requested 2024-12-07T00:51:25,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685905 2024-12-07T00:51:25,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685905 2024-12-07T00:51:25,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685905) roll requested 2024-12-07T00:51:25,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685911 2024-12-07T00:51:25,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685911 2024-12-07T00:51:25,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685911) roll requested 2024-12-07T00:51:25,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685917 2024-12-07T00:51:25,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685917 2024-12-07T00:51:25,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685917) roll requested 2024-12-07T00:51:25,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685923 2024-12-07T00:51:25,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685923 2024-12-07T00:51:25,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685923) roll requested 2024-12-07T00:51:25,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685929 2024-12-07T00:51:25,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685929 2024-12-07T00:51:25,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685929) roll requested 2024-12-07T00:51:25,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685935 2024-12-07T00:51:25,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685935 2024-12-07T00:51:25,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685935) roll requested 2024-12-07T00:51:25,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685940 2024-12-07T00:51:25,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685940 2024-12-07T00:51:25,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685940) roll requested 2024-12-07T00:51:25,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685946 2024-12-07T00:51:25,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685946 2024-12-07T00:51:25,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685946) roll requested 2024-12-07T00:51:25,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685952 2024-12-07T00:51:25,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685952 2024-12-07T00:51:25,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685952) roll requested 2024-12-07T00:51:25,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685958 2024-12-07T00:51:25,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685958 2024-12-07T00:51:25,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685958) roll requested 2024-12-07T00:51:25,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685965 2024-12-07T00:51:25,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685965 2024-12-07T00:51:25,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685965) roll requested 2024-12-07T00:51:25,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685970 2024-12-07T00:51:25,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685970 2024-12-07T00:51:25,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685970) roll requested 2024-12-07T00:51:25,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685975 2024-12-07T00:51:25,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685975 2024-12-07T00:51:25,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685975) roll requested 2024-12-07T00:51:25,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685980 2024-12-07T00:51:25,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685980 2024-12-07T00:51:25,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685980) roll requested 2024-12-07T00:51:25,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685985 2024-12-07T00:51:25,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685985 2024-12-07T00:51:25,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685985) roll requested 2024-12-07T00:51:25,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685990 2024-12-07T00:51:25,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685990 2024-12-07T00:51:25,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:25,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:25,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685990) roll requested 2024-12-07T00:51:25,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532685995 2024-12-07T00:51:25,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:25,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685995 2024-12-07T00:51:26,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532685995) roll requested 2024-12-07T00:51:26,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686000 2024-12-07T00:51:26,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532685995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686000 2024-12-07T00:51:26,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686000) roll requested 2024-12-07T00:51:26,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686005 2024-12-07T00:51:26,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686005 2024-12-07T00:51:26,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686005) roll requested 2024-12-07T00:51:26,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686010 2024-12-07T00:51:26,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686010 2024-12-07T00:51:26,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686010) roll requested 2024-12-07T00:51:26,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686015 2024-12-07T00:51:26,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686015 2024-12-07T00:51:26,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686015) roll requested 2024-12-07T00:51:26,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686020 2024-12-07T00:51:26,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686020 2024-12-07T00:51:26,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686020) roll requested 2024-12-07T00:51:26,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686025 2024-12-07T00:51:26,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686025 2024-12-07T00:51:26,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686025) roll requested 2024-12-07T00:51:26,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686031 2024-12-07T00:51:26,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686031 2024-12-07T00:51:26,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686031) roll requested 2024-12-07T00:51:26,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686041 2024-12-07T00:51:26,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686041 2024-12-07T00:51:26,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686041) roll requested 2024-12-07T00:51:26,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686046 2024-12-07T00:51:26,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686046 2024-12-07T00:51:26,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686046) roll requested 2024-12-07T00:51:26,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686050 2024-12-07T00:51:26,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686050 2024-12-07T00:51:26,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686050) roll requested 2024-12-07T00:51:26,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686056 2024-12-07T00:51:26,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686056 2024-12-07T00:51:26,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686056) roll requested 2024-12-07T00:51:26,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686062 2024-12-07T00:51:26,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686062 2024-12-07T00:51:26,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686062) roll requested 2024-12-07T00:51:26,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686070 2024-12-07T00:51:26,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686070 2024-12-07T00:51:26,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686070) roll requested 2024-12-07T00:51:26,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686078 2024-12-07T00:51:26,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686078 2024-12-07T00:51:26,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686078) roll requested 2024-12-07T00:51:26,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686087 2024-12-07T00:51:26,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686087 2024-12-07T00:51:26,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686087) roll requested 2024-12-07T00:51:26,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686093 2024-12-07T00:51:26,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686093 2024-12-07T00:51:26,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686093) roll requested 2024-12-07T00:51:26,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686100 2024-12-07T00:51:26,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686100 2024-12-07T00:51:26,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686100) roll requested 2024-12-07T00:51:26,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686107 2024-12-07T00:51:26,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686107 2024-12-07T00:51:26,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686107) roll requested 2024-12-07T00:51:26,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686115 2024-12-07T00:51:26,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686115 2024-12-07T00:51:26,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686115) roll requested 2024-12-07T00:51:26,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686125 2024-12-07T00:51:26,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686125 2024-12-07T00:51:26,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686125) roll requested 2024-12-07T00:51:26,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686131 2024-12-07T00:51:26,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686131 2024-12-07T00:51:26,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686131) roll requested 2024-12-07T00:51:26,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686136 2024-12-07T00:51:26,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686136 2024-12-07T00:51:26,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686136) roll requested 2024-12-07T00:51:26,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686141 2024-12-07T00:51:26,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686141 2024-12-07T00:51:26,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686141) roll requested 2024-12-07T00:51:26,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686149 2024-12-07T00:51:26,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686149 2024-12-07T00:51:26,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686149) roll requested 2024-12-07T00:51:26,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686156 2024-12-07T00:51:26,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686156 2024-12-07T00:51:26,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686156) roll requested 2024-12-07T00:51:26,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686163 2024-12-07T00:51:26,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686163 2024-12-07T00:51:26,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686163) roll requested 2024-12-07T00:51:26,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686172 2024-12-07T00:51:26,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686172 2024-12-07T00:51:26,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686172) roll requested 2024-12-07T00:51:26,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686180 2024-12-07T00:51:26,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686180 2024-12-07T00:51:26,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686180) roll requested 2024-12-07T00:51:26,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686187 2024-12-07T00:51:26,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686187 2024-12-07T00:51:26,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686187) roll requested 2024-12-07T00:51:26,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686198 2024-12-07T00:51:26,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686198 2024-12-07T00:51:26,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686198) roll requested 2024-12-07T00:51:26,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686205 2024-12-07T00:51:26,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686205 2024-12-07T00:51:26,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686205) roll requested 2024-12-07T00:51:26,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686213 2024-12-07T00:51:26,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686213 2024-12-07T00:51:26,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686213) roll requested 2024-12-07T00:51:26,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686219 2024-12-07T00:51:26,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686219 2024-12-07T00:51:26,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686219) roll requested 2024-12-07T00:51:26,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686225 2024-12-07T00:51:26,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686225 2024-12-07T00:51:26,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686225) roll requested 2024-12-07T00:51:26,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686231 2024-12-07T00:51:26,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686231 2024-12-07T00:51:26,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686231) roll requested 2024-12-07T00:51:26,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686237 2024-12-07T00:51:26,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686237 2024-12-07T00:51:26,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686237) roll requested 2024-12-07T00:51:26,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686241 2024-12-07T00:51:26,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686241 2024-12-07T00:51:26,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686241) roll requested 2024-12-07T00:51:26,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686246 2024-12-07T00:51:26,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686246 2024-12-07T00:51:26,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686246) roll requested 2024-12-07T00:51:26,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686251 2024-12-07T00:51:26,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686251 2024-12-07T00:51:26,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686251) roll requested 2024-12-07T00:51:26,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686257 2024-12-07T00:51:26,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686257 2024-12-07T00:51:26,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686257) roll requested 2024-12-07T00:51:26,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686264 2024-12-07T00:51:26,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686264 2024-12-07T00:51:26,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686264) roll requested 2024-12-07T00:51:26,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686271 2024-12-07T00:51:26,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686271 2024-12-07T00:51:26,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686271) roll requested 2024-12-07T00:51:26,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686280 2024-12-07T00:51:26,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686280 2024-12-07T00:51:26,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686280) roll requested 2024-12-07T00:51:26,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686288 2024-12-07T00:51:26,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686288 2024-12-07T00:51:26,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686288) roll requested 2024-12-07T00:51:26,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686295 2024-12-07T00:51:26,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686295 2024-12-07T00:51:26,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686295) roll requested 2024-12-07T00:51:26,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686301 2024-12-07T00:51:26,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686301 2024-12-07T00:51:26,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686301) roll requested 2024-12-07T00:51:26,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686308 2024-12-07T00:51:26,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686308 2024-12-07T00:51:26,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686308) roll requested 2024-12-07T00:51:26,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686316 2024-12-07T00:51:26,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686316 2024-12-07T00:51:26,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686316) roll requested 2024-12-07T00:51:26,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686324 2024-12-07T00:51:26,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686324 2024-12-07T00:51:26,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686324) roll requested 2024-12-07T00:51:26,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686332 2024-12-07T00:51:26,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686332 2024-12-07T00:51:26,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686332) roll requested 2024-12-07T00:51:26,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686337 2024-12-07T00:51:26,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686337 2024-12-07T00:51:26,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686337) roll requested 2024-12-07T00:51:26,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686344 2024-12-07T00:51:26,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686344 2024-12-07T00:51:26,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686344) roll requested 2024-12-07T00:51:26,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686351 2024-12-07T00:51:26,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686351 2024-12-07T00:51:26,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686351) roll requested 2024-12-07T00:51:26,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686359 2024-12-07T00:51:26,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686359 2024-12-07T00:51:26,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686359) roll requested 2024-12-07T00:51:26,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686373 2024-12-07T00:51:26,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686373 2024-12-07T00:51:26,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686373) roll requested 2024-12-07T00:51:26,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686378 2024-12-07T00:51:26,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686378 2024-12-07T00:51:26,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686378) roll requested 2024-12-07T00:51:26,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686384 2024-12-07T00:51:26,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686384 2024-12-07T00:51:26,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686384) roll requested 2024-12-07T00:51:26,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686392 2024-12-07T00:51:26,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686392 2024-12-07T00:51:26,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686392) roll requested 2024-12-07T00:51:26,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686397 2024-12-07T00:51:26,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686397 2024-12-07T00:51:26,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686397) roll requested 2024-12-07T00:51:26,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686402 2024-12-07T00:51:26,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686402 2024-12-07T00:51:26,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686402) roll requested 2024-12-07T00:51:26,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686410 2024-12-07T00:51:26,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686410 2024-12-07T00:51:26,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686410) roll requested 2024-12-07T00:51:26,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686419 2024-12-07T00:51:26,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686419 2024-12-07T00:51:26,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686419) roll requested 2024-12-07T00:51:26,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686424 2024-12-07T00:51:26,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686424 2024-12-07T00:51:26,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686424) roll requested 2024-12-07T00:51:26,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686429 2024-12-07T00:51:26,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686429 2024-12-07T00:51:26,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686429) roll requested 2024-12-07T00:51:26,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686436 2024-12-07T00:51:26,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686436 2024-12-07T00:51:26,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686436) roll requested 2024-12-07T00:51:26,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686445 2024-12-07T00:51:26,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686445 2024-12-07T00:51:26,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686445) roll requested 2024-12-07T00:51:26,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686453 2024-12-07T00:51:26,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686453 2024-12-07T00:51:26,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686453) roll requested 2024-12-07T00:51:26,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686459 2024-12-07T00:51:26,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686459 2024-12-07T00:51:26,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686459) roll requested 2024-12-07T00:51:26,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686465 2024-12-07T00:51:26,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686465 2024-12-07T00:51:26,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686465) roll requested 2024-12-07T00:51:26,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686473 2024-12-07T00:51:26,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686473 2024-12-07T00:51:26,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686473) roll requested 2024-12-07T00:51:26,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686482 2024-12-07T00:51:26,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686482 2024-12-07T00:51:26,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686482) roll requested 2024-12-07T00:51:26,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686490 2024-12-07T00:51:26,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686490 2024-12-07T00:51:26,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686490) roll requested 2024-12-07T00:51:26,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686499 2024-12-07T00:51:26,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686499 2024-12-07T00:51:26,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686499) roll requested 2024-12-07T00:51:26,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686508 2024-12-07T00:51:26,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686508 2024-12-07T00:51:26,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686508) roll requested 2024-12-07T00:51:26,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686521 2024-12-07T00:51:26,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686521 2024-12-07T00:51:26,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686521) roll requested 2024-12-07T00:51:26,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686531 2024-12-07T00:51:26,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686531 2024-12-07T00:51:26,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686531) roll requested 2024-12-07T00:51:26,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686537 2024-12-07T00:51:26,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686537 2024-12-07T00:51:26,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686537) roll requested 2024-12-07T00:51:26,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686544 2024-12-07T00:51:26,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686544 2024-12-07T00:51:26,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686544) roll requested 2024-12-07T00:51:26,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686550 2024-12-07T00:51:26,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686550 2024-12-07T00:51:26,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686550) roll requested 2024-12-07T00:51:26,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686555 2024-12-07T00:51:26,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686555 2024-12-07T00:51:26,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686555) roll requested 2024-12-07T00:51:26,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686561 2024-12-07T00:51:26,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686561 2024-12-07T00:51:26,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686561) roll requested 2024-12-07T00:51:26,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686569 2024-12-07T00:51:26,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686569 2024-12-07T00:51:26,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686569) roll requested 2024-12-07T00:51:26,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686578 2024-12-07T00:51:26,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686578 2024-12-07T00:51:26,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686578) roll requested 2024-12-07T00:51:26,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686589 2024-12-07T00:51:26,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686589 2024-12-07T00:51:26,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686589) roll requested 2024-12-07T00:51:26,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686595 2024-12-07T00:51:26,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686595 2024-12-07T00:51:26,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686595) roll requested 2024-12-07T00:51:26,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686600 2024-12-07T00:51:26,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686600 2024-12-07T00:51:26,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686600) roll requested 2024-12-07T00:51:26,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686606 2024-12-07T00:51:26,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686606 2024-12-07T00:51:26,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686606) roll requested 2024-12-07T00:51:26,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686612 2024-12-07T00:51:26,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686612 2024-12-07T00:51:26,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686612) roll requested 2024-12-07T00:51:26,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686620 2024-12-07T00:51:26,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686620 2024-12-07T00:51:26,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686620) roll requested 2024-12-07T00:51:26,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686626 2024-12-07T00:51:26,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686626 2024-12-07T00:51:26,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686626) roll requested 2024-12-07T00:51:26,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686632 2024-12-07T00:51:26,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686632 2024-12-07T00:51:26,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686632) roll requested 2024-12-07T00:51:26,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686639 2024-12-07T00:51:26,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686639 2024-12-07T00:51:26,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686639) roll requested 2024-12-07T00:51:26,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686646 2024-12-07T00:51:26,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686646 2024-12-07T00:51:26,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686646) roll requested 2024-12-07T00:51:26,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686654 2024-12-07T00:51:26,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686654 2024-12-07T00:51:26,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686654) roll requested 2024-12-07T00:51:26,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686662 2024-12-07T00:51:26,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686662 2024-12-07T00:51:26,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686662) roll requested 2024-12-07T00:51:26,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686670 2024-12-07T00:51:26,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686670 2024-12-07T00:51:26,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686670) roll requested 2024-12-07T00:51:26,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686678 2024-12-07T00:51:26,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686678 2024-12-07T00:51:26,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686678) roll requested 2024-12-07T00:51:26,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686688 2024-12-07T00:51:26,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686688 2024-12-07T00:51:26,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686688) roll requested 2024-12-07T00:51:26,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686693 2024-12-07T00:51:26,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686693 2024-12-07T00:51:26,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686693) roll requested 2024-12-07T00:51:26,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686698 2024-12-07T00:51:26,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686698 2024-12-07T00:51:26,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686698) roll requested 2024-12-07T00:51:26,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686704 2024-12-07T00:51:26,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686704 2024-12-07T00:51:26,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686704) roll requested 2024-12-07T00:51:26,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686711 2024-12-07T00:51:26,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686711 2024-12-07T00:51:26,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686711) roll requested 2024-12-07T00:51:26,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686719 2024-12-07T00:51:26,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686719 2024-12-07T00:51:26,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686719) roll requested 2024-12-07T00:51:26,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686727 2024-12-07T00:51:26,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686727 2024-12-07T00:51:26,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686727) roll requested 2024-12-07T00:51:26,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686736 2024-12-07T00:51:26,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686736 2024-12-07T00:51:26,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686736) roll requested 2024-12-07T00:51:26,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686744 2024-12-07T00:51:26,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686744 2024-12-07T00:51:26,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686744) roll requested 2024-12-07T00:51:26,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686751 2024-12-07T00:51:26,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686751 2024-12-07T00:51:26,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686751) roll requested 2024-12-07T00:51:26,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686760 2024-12-07T00:51:26,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686760 2024-12-07T00:51:26,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686760) roll requested 2024-12-07T00:51:26,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686766 2024-12-07T00:51:26,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686766 2024-12-07T00:51:26,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686766) roll requested 2024-12-07T00:51:26,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686773 2024-12-07T00:51:26,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686773 2024-12-07T00:51:26,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686773) roll requested 2024-12-07T00:51:26,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686785 2024-12-07T00:51:26,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686785 2024-12-07T00:51:26,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686785) roll requested 2024-12-07T00:51:26,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686792 2024-12-07T00:51:26,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686792 2024-12-07T00:51:26,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686792) roll requested 2024-12-07T00:51:26,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686798 2024-12-07T00:51:26,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686798 2024-12-07T00:51:26,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686798) roll requested 2024-12-07T00:51:26,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686803 2024-12-07T00:51:26,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686803 2024-12-07T00:51:26,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686803) roll requested 2024-12-07T00:51:26,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686808 2024-12-07T00:51:26,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686808 2024-12-07T00:51:26,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686808) roll requested 2024-12-07T00:51:26,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686813 2024-12-07T00:51:26,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686813 2024-12-07T00:51:26,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686813) roll requested 2024-12-07T00:51:26,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686818 2024-12-07T00:51:26,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686818 2024-12-07T00:51:26,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686818) roll requested 2024-12-07T00:51:26,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686823 2024-12-07T00:51:26,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686823 2024-12-07T00:51:26,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686823) roll requested 2024-12-07T00:51:26,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686829 2024-12-07T00:51:26,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686829 2024-12-07T00:51:26,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686829) roll requested 2024-12-07T00:51:26,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686835 2024-12-07T00:51:26,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686835 2024-12-07T00:51:26,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686835) roll requested 2024-12-07T00:51:26,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686843 2024-12-07T00:51:26,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686843 2024-12-07T00:51:26,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686843) roll requested 2024-12-07T00:51:26,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686849 2024-12-07T00:51:26,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686849 2024-12-07T00:51:26,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686849) roll requested 2024-12-07T00:51:26,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686856 2024-12-07T00:51:26,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686856 2024-12-07T00:51:26,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686856) roll requested 2024-12-07T00:51:26,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686861 2024-12-07T00:51:26,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686861 2024-12-07T00:51:26,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686861) roll requested 2024-12-07T00:51:26,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686867 2024-12-07T00:51:26,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686867 2024-12-07T00:51:26,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686867) roll requested 2024-12-07T00:51:26,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686872 2024-12-07T00:51:26,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686872 2024-12-07T00:51:26,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686872) roll requested 2024-12-07T00:51:26,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686878 2024-12-07T00:51:26,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686878 2024-12-07T00:51:26,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686878) roll requested 2024-12-07T00:51:26,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686885 2024-12-07T00:51:26,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686885 2024-12-07T00:51:26,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686885) roll requested 2024-12-07T00:51:26,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686891 2024-12-07T00:51:26,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686891 2024-12-07T00:51:26,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686891) roll requested 2024-12-07T00:51:26,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686896 2024-12-07T00:51:26,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686896 2024-12-07T00:51:26,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686896) roll requested 2024-12-07T00:51:26,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686902 2024-12-07T00:51:26,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686902 2024-12-07T00:51:26,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686902) roll requested 2024-12-07T00:51:26,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686908 2024-12-07T00:51:26,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686908 2024-12-07T00:51:26,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686908) roll requested 2024-12-07T00:51:26,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686913 2024-12-07T00:51:26,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686913 2024-12-07T00:51:26,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686913) roll requested 2024-12-07T00:51:26,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686919 2024-12-07T00:51:26,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686919 2024-12-07T00:51:26,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686919) roll requested 2024-12-07T00:51:26,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686924 2024-12-07T00:51:26,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686924 2024-12-07T00:51:26,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686924) roll requested 2024-12-07T00:51:26,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686930 2024-12-07T00:51:26,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686930 2024-12-07T00:51:26,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686930) roll requested 2024-12-07T00:51:26,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686936 2024-12-07T00:51:26,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686936 2024-12-07T00:51:26,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686936) roll requested 2024-12-07T00:51:26,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686949 2024-12-07T00:51:26,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686949 2024-12-07T00:51:26,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686949) roll requested 2024-12-07T00:51:26,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686955 2024-12-07T00:51:26,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686955 2024-12-07T00:51:26,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686955) roll requested 2024-12-07T00:51:26,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686960 2024-12-07T00:51:26,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686960 2024-12-07T00:51:26,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686960) roll requested 2024-12-07T00:51:26,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686965 2024-12-07T00:51:26,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686965 2024-12-07T00:51:26,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686965) roll requested 2024-12-07T00:51:26,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686971 2024-12-07T00:51:26,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686971 2024-12-07T00:51:26,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686971) roll requested 2024-12-07T00:51:26,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686977 2024-12-07T00:51:26,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686977 2024-12-07T00:51:26,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686977) roll requested 2024-12-07T00:51:26,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686983 2024-12-07T00:51:26,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686983 2024-12-07T00:51:26,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686983) roll requested 2024-12-07T00:51:26,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686989 2024-12-07T00:51:26,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686989 2024-12-07T00:51:26,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:26,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:26,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686989) roll requested 2024-12-07T00:51:26,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532686995 2024-12-07T00:51:26,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:26,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686995 2024-12-07T00:51:27,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532686995) roll requested 2024-12-07T00:51:27,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687000 2024-12-07T00:51:27,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532686995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687000 2024-12-07T00:51:27,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687000) roll requested 2024-12-07T00:51:27,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687006 2024-12-07T00:51:27,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687006 2024-12-07T00:51:27,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687006) roll requested 2024-12-07T00:51:27,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687012 2024-12-07T00:51:27,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687012 2024-12-07T00:51:27,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687012) roll requested 2024-12-07T00:51:27,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687017 2024-12-07T00:51:27,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687017 2024-12-07T00:51:27,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687017) roll requested 2024-12-07T00:51:27,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687023 2024-12-07T00:51:27,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687023 2024-12-07T00:51:27,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687023) roll requested 2024-12-07T00:51:27,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687028 2024-12-07T00:51:27,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687028 2024-12-07T00:51:27,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687028) roll requested 2024-12-07T00:51:27,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687036 2024-12-07T00:51:27,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687036 2024-12-07T00:51:27,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687036) roll requested 2024-12-07T00:51:27,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687042 2024-12-07T00:51:27,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687042 2024-12-07T00:51:27,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687042) roll requested 2024-12-07T00:51:27,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687048 2024-12-07T00:51:27,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687048 2024-12-07T00:51:27,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687048) roll requested 2024-12-07T00:51:27,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687055 2024-12-07T00:51:27,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687055 2024-12-07T00:51:27,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687055) roll requested 2024-12-07T00:51:27,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687062 2024-12-07T00:51:27,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687062 2024-12-07T00:51:27,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687062) roll requested 2024-12-07T00:51:27,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687070 2024-12-07T00:51:27,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687070 2024-12-07T00:51:27,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687070) roll requested 2024-12-07T00:51:27,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687076 2024-12-07T00:51:27,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687076 2024-12-07T00:51:27,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687076) roll requested 2024-12-07T00:51:27,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687084 2024-12-07T00:51:27,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687084 2024-12-07T00:51:27,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687084) roll requested 2024-12-07T00:51:27,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687092 2024-12-07T00:51:27,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687092 2024-12-07T00:51:27,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687092) roll requested 2024-12-07T00:51:27,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687100 2024-12-07T00:51:27,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687100 2024-12-07T00:51:27,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687100) roll requested 2024-12-07T00:51:27,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687107 2024-12-07T00:51:27,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687107 2024-12-07T00:51:27,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687107) roll requested 2024-12-07T00:51:27,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687116 2024-12-07T00:51:27,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687116 2024-12-07T00:51:27,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687116) roll requested 2024-12-07T00:51:27,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687129 2024-12-07T00:51:27,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687129 2024-12-07T00:51:27,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687129) roll requested 2024-12-07T00:51:27,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687136 2024-12-07T00:51:27,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687136 2024-12-07T00:51:27,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687136) roll requested 2024-12-07T00:51:27,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687141 2024-12-07T00:51:27,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687141 2024-12-07T00:51:27,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687141) roll requested 2024-12-07T00:51:27,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687148 2024-12-07T00:51:27,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687148 2024-12-07T00:51:27,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687148) roll requested 2024-12-07T00:51:27,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687155 2024-12-07T00:51:27,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687155 2024-12-07T00:51:27,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687155) roll requested 2024-12-07T00:51:27,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687160 2024-12-07T00:51:27,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687160 2024-12-07T00:51:27,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687160) roll requested 2024-12-07T00:51:27,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687165 2024-12-07T00:51:27,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687165 2024-12-07T00:51:27,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687165) roll requested 2024-12-07T00:51:27,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687171 2024-12-07T00:51:27,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687171 2024-12-07T00:51:27,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687171) roll requested 2024-12-07T00:51:27,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687176 2024-12-07T00:51:27,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687176 2024-12-07T00:51:27,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687176) roll requested 2024-12-07T00:51:27,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687183 2024-12-07T00:51:27,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687183 2024-12-07T00:51:27,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687183) roll requested 2024-12-07T00:51:27,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687190 2024-12-07T00:51:27,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687190 2024-12-07T00:51:27,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687190) roll requested 2024-12-07T00:51:27,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687196 2024-12-07T00:51:27,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687196 2024-12-07T00:51:27,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687196) roll requested 2024-12-07T00:51:27,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687201 2024-12-07T00:51:27,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687201 2024-12-07T00:51:27,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687201) roll requested 2024-12-07T00:51:27,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687206 2024-12-07T00:51:27,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687206 2024-12-07T00:51:27,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687206) roll requested 2024-12-07T00:51:27,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687212 2024-12-07T00:51:27,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687212 2024-12-07T00:51:27,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687212) roll requested 2024-12-07T00:51:27,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687217 2024-12-07T00:51:27,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687217 2024-12-07T00:51:27,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687217) roll requested 2024-12-07T00:51:27,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687222 2024-12-07T00:51:27,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687222 2024-12-07T00:51:27,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687222) roll requested 2024-12-07T00:51:27,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687227 2024-12-07T00:51:27,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687227 2024-12-07T00:51:27,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687227) roll requested 2024-12-07T00:51:27,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687233 2024-12-07T00:51:27,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687233 2024-12-07T00:51:27,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687233) roll requested 2024-12-07T00:51:27,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687238 2024-12-07T00:51:27,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687238 2024-12-07T00:51:27,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687238) roll requested 2024-12-07T00:51:27,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687244 2024-12-07T00:51:27,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687244 2024-12-07T00:51:27,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687244) roll requested 2024-12-07T00:51:27,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687249 2024-12-07T00:51:27,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687249 2024-12-07T00:51:27,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687249) roll requested 2024-12-07T00:51:27,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687254 2024-12-07T00:51:27,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687254 2024-12-07T00:51:27,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687254) roll requested 2024-12-07T00:51:27,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687260 2024-12-07T00:51:27,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687260 2024-12-07T00:51:27,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687260) roll requested 2024-12-07T00:51:27,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687265 2024-12-07T00:51:27,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687265 2024-12-07T00:51:27,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687265) roll requested 2024-12-07T00:51:27,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687271 2024-12-07T00:51:27,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687271 2024-12-07T00:51:27,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687271) roll requested 2024-12-07T00:51:27,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687278 2024-12-07T00:51:27,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687278 2024-12-07T00:51:27,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687278) roll requested 2024-12-07T00:51:27,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687283 2024-12-07T00:51:27,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687283 2024-12-07T00:51:27,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687283) roll requested 2024-12-07T00:51:27,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687288 2024-12-07T00:51:27,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687288 2024-12-07T00:51:27,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687288) roll requested 2024-12-07T00:51:27,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687294 2024-12-07T00:51:27,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687294 2024-12-07T00:51:27,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687294) roll requested 2024-12-07T00:51:27,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687299 2024-12-07T00:51:27,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687299 2024-12-07T00:51:27,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687299) roll requested 2024-12-07T00:51:27,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687306 2024-12-07T00:51:27,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687306 2024-12-07T00:51:27,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687306) roll requested 2024-12-07T00:51:27,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687312 2024-12-07T00:51:27,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687312 2024-12-07T00:51:27,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687312) roll requested 2024-12-07T00:51:27,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687318 2024-12-07T00:51:27,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687318 2024-12-07T00:51:27,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687318) roll requested 2024-12-07T00:51:27,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687325 2024-12-07T00:51:27,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687325 2024-12-07T00:51:27,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687325) roll requested 2024-12-07T00:51:27,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687331 2024-12-07T00:51:27,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687331 2024-12-07T00:51:27,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687331) roll requested 2024-12-07T00:51:27,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687338 2024-12-07T00:51:27,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687338 2024-12-07T00:51:27,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687338) roll requested 2024-12-07T00:51:27,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687344 2024-12-07T00:51:27,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687344 2024-12-07T00:51:27,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687344) roll requested 2024-12-07T00:51:27,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687350 2024-12-07T00:51:27,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687350 2024-12-07T00:51:27,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687350) roll requested 2024-12-07T00:51:27,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687355 2024-12-07T00:51:27,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687355 2024-12-07T00:51:27,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687355) roll requested 2024-12-07T00:51:27,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687361 2024-12-07T00:51:27,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687361 2024-12-07T00:51:27,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687361) roll requested 2024-12-07T00:51:27,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687366 2024-12-07T00:51:27,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687366 2024-12-07T00:51:27,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687366) roll requested 2024-12-07T00:51:27,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687371 2024-12-07T00:51:27,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687371 2024-12-07T00:51:27,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687371) roll requested 2024-12-07T00:51:27,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687376 2024-12-07T00:51:27,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687376 2024-12-07T00:51:27,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687376) roll requested 2024-12-07T00:51:27,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687382 2024-12-07T00:51:27,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687382 2024-12-07T00:51:27,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687382) roll requested 2024-12-07T00:51:27,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687389 2024-12-07T00:51:27,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687389 2024-12-07T00:51:27,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687389) roll requested 2024-12-07T00:51:27,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687398 2024-12-07T00:51:27,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687398 2024-12-07T00:51:27,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687398) roll requested 2024-12-07T00:51:27,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687408 2024-12-07T00:51:27,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687408 2024-12-07T00:51:27,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687408) roll requested 2024-12-07T00:51:27,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687418 2024-12-07T00:51:27,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687418 2024-12-07T00:51:27,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687418) roll requested 2024-12-07T00:51:27,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687425 2024-12-07T00:51:27,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687425 2024-12-07T00:51:27,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687425) roll requested 2024-12-07T00:51:27,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687430 2024-12-07T00:51:27,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687430 2024-12-07T00:51:27,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687430) roll requested 2024-12-07T00:51:27,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687436 2024-12-07T00:51:27,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687436 2024-12-07T00:51:27,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687436) roll requested 2024-12-07T00:51:27,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687441 2024-12-07T00:51:27,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687441 2024-12-07T00:51:27,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687441) roll requested 2024-12-07T00:51:27,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687447 2024-12-07T00:51:27,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687447 2024-12-07T00:51:27,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687447) roll requested 2024-12-07T00:51:27,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687453 2024-12-07T00:51:27,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687453 2024-12-07T00:51:27,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687453) roll requested 2024-12-07T00:51:27,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687458 2024-12-07T00:51:27,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687458 2024-12-07T00:51:27,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687458) roll requested 2024-12-07T00:51:27,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687463 2024-12-07T00:51:27,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687463 2024-12-07T00:51:27,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687463) roll requested 2024-12-07T00:51:27,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687468 2024-12-07T00:51:27,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687468 2024-12-07T00:51:27,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687468) roll requested 2024-12-07T00:51:27,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687473 2024-12-07T00:51:27,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687473 2024-12-07T00:51:27,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687473) roll requested 2024-12-07T00:51:27,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687479 2024-12-07T00:51:27,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687479 2024-12-07T00:51:27,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687479) roll requested 2024-12-07T00:51:27,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687484 2024-12-07T00:51:27,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687484 2024-12-07T00:51:27,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687484) roll requested 2024-12-07T00:51:27,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687490 2024-12-07T00:51:27,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687490 2024-12-07T00:51:27,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687490) roll requested 2024-12-07T00:51:27,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687495 2024-12-07T00:51:27,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687495 2024-12-07T00:51:27,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687495) roll requested 2024-12-07T00:51:27,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687500 2024-12-07T00:51:27,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687500 2024-12-07T00:51:27,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687500) roll requested 2024-12-07T00:51:27,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687505 2024-12-07T00:51:27,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687505 2024-12-07T00:51:27,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687505) roll requested 2024-12-07T00:51:27,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687510 2024-12-07T00:51:27,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687510 2024-12-07T00:51:27,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687510) roll requested 2024-12-07T00:51:27,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687515 2024-12-07T00:51:27,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687515 2024-12-07T00:51:27,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687515) roll requested 2024-12-07T00:51:27,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687520 2024-12-07T00:51:27,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687520 2024-12-07T00:51:27,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687520) roll requested 2024-12-07T00:51:27,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687526 2024-12-07T00:51:27,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687526 2024-12-07T00:51:27,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687526) roll requested 2024-12-07T00:51:27,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687532 2024-12-07T00:51:27,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687532 2024-12-07T00:51:27,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687532) roll requested 2024-12-07T00:51:27,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687537 2024-12-07T00:51:27,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687537 2024-12-07T00:51:27,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687537) roll requested 2024-12-07T00:51:27,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687543 2024-12-07T00:51:27,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687543 2024-12-07T00:51:27,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687543) roll requested 2024-12-07T00:51:27,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687549 2024-12-07T00:51:27,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687549 2024-12-07T00:51:27,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687549) roll requested 2024-12-07T00:51:27,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687560 2024-12-07T00:51:27,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687560 2024-12-07T00:51:27,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687560) roll requested 2024-12-07T00:51:27,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687571 2024-12-07T00:51:27,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687571 2024-12-07T00:51:27,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687571) roll requested 2024-12-07T00:51:27,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687577 2024-12-07T00:51:27,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687577 2024-12-07T00:51:27,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687577) roll requested 2024-12-07T00:51:27,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687582 2024-12-07T00:51:27,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687582 2024-12-07T00:51:27,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687582) roll requested 2024-12-07T00:51:27,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687587 2024-12-07T00:51:27,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687587 2024-12-07T00:51:27,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687587) roll requested 2024-12-07T00:51:27,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687593 2024-12-07T00:51:27,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687593 2024-12-07T00:51:27,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687593) roll requested 2024-12-07T00:51:27,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687598 2024-12-07T00:51:27,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687598 2024-12-07T00:51:27,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687598) roll requested 2024-12-07T00:51:27,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687603 2024-12-07T00:51:27,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687603 2024-12-07T00:51:27,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687603) roll requested 2024-12-07T00:51:27,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687608 2024-12-07T00:51:27,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687608 2024-12-07T00:51:27,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687608) roll requested 2024-12-07T00:51:27,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687613 2024-12-07T00:51:27,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687613 2024-12-07T00:51:27,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687613) roll requested 2024-12-07T00:51:27,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687619 2024-12-07T00:51:27,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687619 2024-12-07T00:51:27,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687619) roll requested 2024-12-07T00:51:27,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687632 2024-12-07T00:51:27,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687632 2024-12-07T00:51:27,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687632) roll requested 2024-12-07T00:51:27,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687648 2024-12-07T00:51:27,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687648 2024-12-07T00:51:27,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687648) roll requested 2024-12-07T00:51:27,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687654 2024-12-07T00:51:27,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687654 2024-12-07T00:51:27,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687654) roll requested 2024-12-07T00:51:27,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687661 2024-12-07T00:51:27,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687661 2024-12-07T00:51:27,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687661) roll requested 2024-12-07T00:51:27,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687667 2024-12-07T00:51:27,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687667 2024-12-07T00:51:27,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687667) roll requested 2024-12-07T00:51:27,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687672 2024-12-07T00:51:27,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687672 2024-12-07T00:51:27,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687672) roll requested 2024-12-07T00:51:27,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687678 2024-12-07T00:51:27,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687678 2024-12-07T00:51:27,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687678) roll requested 2024-12-07T00:51:27,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687683 2024-12-07T00:51:27,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687683 2024-12-07T00:51:27,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687683) roll requested 2024-12-07T00:51:27,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687689 2024-12-07T00:51:27,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687689 2024-12-07T00:51:27,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687689) roll requested 2024-12-07T00:51:27,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687695 2024-12-07T00:51:27,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687695 2024-12-07T00:51:27,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687695) roll requested 2024-12-07T00:51:27,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687702 2024-12-07T00:51:27,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687702 2024-12-07T00:51:27,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687702) roll requested 2024-12-07T00:51:27,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687708 2024-12-07T00:51:27,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687708 2024-12-07T00:51:27,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687708) roll requested 2024-12-07T00:51:27,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687715 2024-12-07T00:51:27,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687715 2024-12-07T00:51:27,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687715) roll requested 2024-12-07T00:51:27,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687720 2024-12-07T00:51:27,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687720 2024-12-07T00:51:27,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687720) roll requested 2024-12-07T00:51:27,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687726 2024-12-07T00:51:27,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687726 2024-12-07T00:51:27,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687726) roll requested 2024-12-07T00:51:27,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687732 2024-12-07T00:51:27,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687732 2024-12-07T00:51:27,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687732) roll requested 2024-12-07T00:51:27,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687737 2024-12-07T00:51:27,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687737 2024-12-07T00:51:27,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687737) roll requested 2024-12-07T00:51:27,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687742 2024-12-07T00:51:27,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687742 2024-12-07T00:51:27,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687742) roll requested 2024-12-07T00:51:27,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687747 2024-12-07T00:51:27,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687747 2024-12-07T00:51:27,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687747) roll requested 2024-12-07T00:51:27,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687752 2024-12-07T00:51:27,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687752 2024-12-07T00:51:27,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687752) roll requested 2024-12-07T00:51:27,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687758 2024-12-07T00:51:27,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687758 2024-12-07T00:51:27,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687758) roll requested 2024-12-07T00:51:27,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687763 2024-12-07T00:51:27,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687763 2024-12-07T00:51:27,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687763) roll requested 2024-12-07T00:51:27,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687771 2024-12-07T00:51:27,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687771 2024-12-07T00:51:27,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687771) roll requested 2024-12-07T00:51:27,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687776 2024-12-07T00:51:27,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687776 2024-12-07T00:51:27,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687776) roll requested 2024-12-07T00:51:27,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687783 2024-12-07T00:51:27,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687783 2024-12-07T00:51:27,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687783) roll requested 2024-12-07T00:51:27,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687788 2024-12-07T00:51:27,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687788 2024-12-07T00:51:27,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687788) roll requested 2024-12-07T00:51:27,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687795 2024-12-07T00:51:27,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687795 2024-12-07T00:51:27,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687795) roll requested 2024-12-07T00:51:27,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687801 2024-12-07T00:51:27,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687801 2024-12-07T00:51:27,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687801) roll requested 2024-12-07T00:51:27,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687809 2024-12-07T00:51:27,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687809 2024-12-07T00:51:27,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687809) roll requested 2024-12-07T00:51:27,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687816 2024-12-07T00:51:27,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687816 2024-12-07T00:51:27,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687816) roll requested 2024-12-07T00:51:27,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687822 2024-12-07T00:51:27,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687822 2024-12-07T00:51:27,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687822) roll requested 2024-12-07T00:51:27,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687828 2024-12-07T00:51:27,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687828 2024-12-07T00:51:27,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687828) roll requested 2024-12-07T00:51:27,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687834 2024-12-07T00:51:27,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687834 2024-12-07T00:51:27,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687834) roll requested 2024-12-07T00:51:27,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687841 2024-12-07T00:51:27,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687841 2024-12-07T00:51:27,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687841) roll requested 2024-12-07T00:51:27,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687848 2024-12-07T00:51:27,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687848 2024-12-07T00:51:27,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687848) roll requested 2024-12-07T00:51:27,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687855 2024-12-07T00:51:27,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687855 2024-12-07T00:51:27,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687855) roll requested 2024-12-07T00:51:27,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687859 2024-12-07T00:51:27,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687859 2024-12-07T00:51:27,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687859) roll requested 2024-12-07T00:51:27,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687864 2024-12-07T00:51:27,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687864 2024-12-07T00:51:27,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687864) roll requested 2024-12-07T00:51:27,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687870 2024-12-07T00:51:27,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687870 2024-12-07T00:51:27,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687870) roll requested 2024-12-07T00:51:27,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687874 2024-12-07T00:51:27,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687874 2024-12-07T00:51:27,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687874) roll requested 2024-12-07T00:51:27,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687879 2024-12-07T00:51:27,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687879 2024-12-07T00:51:27,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687879) roll requested 2024-12-07T00:51:27,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687885 2024-12-07T00:51:27,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687885 2024-12-07T00:51:27,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687885) roll requested 2024-12-07T00:51:27,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687892 2024-12-07T00:51:27,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687892 2024-12-07T00:51:27,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687892) roll requested 2024-12-07T00:51:27,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687899 2024-12-07T00:51:27,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687899 2024-12-07T00:51:27,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687899) roll requested 2024-12-07T00:51:27,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687908 2024-12-07T00:51:27,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687908 2024-12-07T00:51:27,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687908) roll requested 2024-12-07T00:51:27,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687914 2024-12-07T00:51:27,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687914 2024-12-07T00:51:27,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687914) roll requested 2024-12-07T00:51:27,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687919 2024-12-07T00:51:27,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687919 2024-12-07T00:51:27,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687919) roll requested 2024-12-07T00:51:27,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687925 2024-12-07T00:51:27,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687925 2024-12-07T00:51:27,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687925) roll requested 2024-12-07T00:51:27,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687932 2024-12-07T00:51:27,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687932 2024-12-07T00:51:27,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687932) roll requested 2024-12-07T00:51:27,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687941 2024-12-07T00:51:27,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687941 2024-12-07T00:51:27,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687941) roll requested 2024-12-07T00:51:27,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687946 2024-12-07T00:51:27,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687946 2024-12-07T00:51:27,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687946) roll requested 2024-12-07T00:51:27,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687952 2024-12-07T00:51:27,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687952 2024-12-07T00:51:27,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687952) roll requested 2024-12-07T00:51:27,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687959 2024-12-07T00:51:27,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687959 2024-12-07T00:51:27,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687959) roll requested 2024-12-07T00:51:27,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687967 2024-12-07T00:51:27,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687967 2024-12-07T00:51:27,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687967) roll requested 2024-12-07T00:51:27,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687974 2024-12-07T00:51:27,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687974 2024-12-07T00:51:27,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687974) roll requested 2024-12-07T00:51:27,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687978 2024-12-07T00:51:27,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687978 2024-12-07T00:51:27,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687978) roll requested 2024-12-07T00:51:27,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687986 2024-12-07T00:51:27,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:27,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687986 2024-12-07T00:51:27,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:27,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:27,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687986) roll requested 2024-12-07T00:51:27,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532687996 2024-12-07T00:51:28,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687996 2024-12-07T00:51:28,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532687996) roll requested 2024-12-07T00:51:28,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688004 2024-12-07T00:51:28,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532687996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688004 2024-12-07T00:51:28,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688004) roll requested 2024-12-07T00:51:28,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688011 2024-12-07T00:51:28,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688011 2024-12-07T00:51:28,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688011) roll requested 2024-12-07T00:51:28,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688017 2024-12-07T00:51:28,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688017 2024-12-07T00:51:28,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688017) roll requested 2024-12-07T00:51:28,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688022 2024-12-07T00:51:28,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688022 2024-12-07T00:51:28,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688022) roll requested 2024-12-07T00:51:28,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688030 2024-12-07T00:51:28,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688030 2024-12-07T00:51:28,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688030) roll requested 2024-12-07T00:51:28,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688035 2024-12-07T00:51:28,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688035 2024-12-07T00:51:28,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688035) roll requested 2024-12-07T00:51:28,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688041 2024-12-07T00:51:28,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688041 2024-12-07T00:51:28,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688041) roll requested 2024-12-07T00:51:28,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688047 2024-12-07T00:51:28,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688047 2024-12-07T00:51:28,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688047) roll requested 2024-12-07T00:51:28,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688052 2024-12-07T00:51:28,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688052 2024-12-07T00:51:28,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688052) roll requested 2024-12-07T00:51:28,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688057 2024-12-07T00:51:28,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688057 2024-12-07T00:51:28,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688057) roll requested 2024-12-07T00:51:28,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688063 2024-12-07T00:51:28,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688063 2024-12-07T00:51:28,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688063) roll requested 2024-12-07T00:51:28,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688068 2024-12-07T00:51:28,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688068 2024-12-07T00:51:28,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688068) roll requested 2024-12-07T00:51:28,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688073 2024-12-07T00:51:28,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688073 2024-12-07T00:51:28,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688073) roll requested 2024-12-07T00:51:28,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688079 2024-12-07T00:51:28,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688079 2024-12-07T00:51:28,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688079) roll requested 2024-12-07T00:51:28,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688085 2024-12-07T00:51:28,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688085 2024-12-07T00:51:28,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688085) roll requested 2024-12-07T00:51:28,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688090 2024-12-07T00:51:28,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688090 2024-12-07T00:51:28,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688090) roll requested 2024-12-07T00:51:28,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688096 2024-12-07T00:51:28,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688096 2024-12-07T00:51:28,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688096) roll requested 2024-12-07T00:51:28,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688101 2024-12-07T00:51:28,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688101 2024-12-07T00:51:28,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688101) roll requested 2024-12-07T00:51:28,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688106 2024-12-07T00:51:28,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688106 2024-12-07T00:51:28,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688106) roll requested 2024-12-07T00:51:28,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688112 2024-12-07T00:51:28,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688112 2024-12-07T00:51:28,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688112) roll requested 2024-12-07T00:51:28,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688117 2024-12-07T00:51:28,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688117 2024-12-07T00:51:28,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688117) roll requested 2024-12-07T00:51:28,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688123 2024-12-07T00:51:28,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688123 2024-12-07T00:51:28,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688123) roll requested 2024-12-07T00:51:28,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688129 2024-12-07T00:51:28,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688129 2024-12-07T00:51:28,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688129) roll requested 2024-12-07T00:51:28,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688134 2024-12-07T00:51:28,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688134 2024-12-07T00:51:28,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688134) roll requested 2024-12-07T00:51:28,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688141 2024-12-07T00:51:28,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688141 2024-12-07T00:51:28,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688141) roll requested 2024-12-07T00:51:28,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688147 2024-12-07T00:51:28,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688147 2024-12-07T00:51:28,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688147) roll requested 2024-12-07T00:51:28,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688152 2024-12-07T00:51:28,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688152 2024-12-07T00:51:28,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688152) roll requested 2024-12-07T00:51:28,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688158 2024-12-07T00:51:28,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688158 2024-12-07T00:51:28,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688158) roll requested 2024-12-07T00:51:28,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688164 2024-12-07T00:51:28,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688164 2024-12-07T00:51:28,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688164) roll requested 2024-12-07T00:51:28,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688170 2024-12-07T00:51:28,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688170 2024-12-07T00:51:28,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688170) roll requested 2024-12-07T00:51:28,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688176 2024-12-07T00:51:28,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688176 2024-12-07T00:51:28,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688176) roll requested 2024-12-07T00:51:28,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688181 2024-12-07T00:51:28,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688181 2024-12-07T00:51:28,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688181) roll requested 2024-12-07T00:51:28,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688186 2024-12-07T00:51:28,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688186 2024-12-07T00:51:28,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688186) roll requested 2024-12-07T00:51:28,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688193 2024-12-07T00:51:28,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688193 2024-12-07T00:51:28,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688193) roll requested 2024-12-07T00:51:28,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688199 2024-12-07T00:51:28,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688199 2024-12-07T00:51:28,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688199) roll requested 2024-12-07T00:51:28,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688204 2024-12-07T00:51:28,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688204 2024-12-07T00:51:28,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688204) roll requested 2024-12-07T00:51:28,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688210 2024-12-07T00:51:28,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688210 2024-12-07T00:51:28,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688210) roll requested 2024-12-07T00:51:28,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688216 2024-12-07T00:51:28,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688216 2024-12-07T00:51:28,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688216) roll requested 2024-12-07T00:51:28,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688221 2024-12-07T00:51:28,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688221 2024-12-07T00:51:28,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688221) roll requested 2024-12-07T00:51:28,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688227 2024-12-07T00:51:28,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688227 2024-12-07T00:51:28,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688227) roll requested 2024-12-07T00:51:28,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688233 2024-12-07T00:51:28,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688233 2024-12-07T00:51:28,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688233) roll requested 2024-12-07T00:51:28,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688239 2024-12-07T00:51:28,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688239 2024-12-07T00:51:28,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688239) roll requested 2024-12-07T00:51:28,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688244 2024-12-07T00:51:28,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688244 2024-12-07T00:51:28,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688244) roll requested 2024-12-07T00:51:28,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688250 2024-12-07T00:51:28,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688250 2024-12-07T00:51:28,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688250) roll requested 2024-12-07T00:51:28,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688256 2024-12-07T00:51:28,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688256 2024-12-07T00:51:28,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688256) roll requested 2024-12-07T00:51:28,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688261 2024-12-07T00:51:28,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688261 2024-12-07T00:51:28,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688261) roll requested 2024-12-07T00:51:28,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688267 2024-12-07T00:51:28,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688267 2024-12-07T00:51:28,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688267) roll requested 2024-12-07T00:51:28,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688273 2024-12-07T00:51:28,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688273 2024-12-07T00:51:28,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688273) roll requested 2024-12-07T00:51:28,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688279 2024-12-07T00:51:28,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688279 2024-12-07T00:51:28,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688279) roll requested 2024-12-07T00:51:28,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688284 2024-12-07T00:51:28,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688284 2024-12-07T00:51:28,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688284) roll requested 2024-12-07T00:51:28,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688290 2024-12-07T00:51:28,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688290 2024-12-07T00:51:28,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688290) roll requested 2024-12-07T00:51:28,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688295 2024-12-07T00:51:28,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688295 2024-12-07T00:51:28,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688295) roll requested 2024-12-07T00:51:28,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688299 2024-12-07T00:51:28,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688299 2024-12-07T00:51:28,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688299) roll requested 2024-12-07T00:51:28,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688307 2024-12-07T00:51:28,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688307 2024-12-07T00:51:28,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688307) roll requested 2024-12-07T00:51:28,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688314 2024-12-07T00:51:28,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688314 2024-12-07T00:51:28,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688314) roll requested 2024-12-07T00:51:28,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688377 2024-12-07T00:51:28,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688377 2024-12-07T00:51:28,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688377) roll requested 2024-12-07T00:51:28,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688387 2024-12-07T00:51:28,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688387 2024-12-07T00:51:28,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688387) roll requested 2024-12-07T00:51:28,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688397 2024-12-07T00:51:28,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688397 2024-12-07T00:51:28,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688397) roll requested 2024-12-07T00:51:28,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688407 2024-12-07T00:51:28,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688407 2024-12-07T00:51:28,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688407) roll requested 2024-12-07T00:51:28,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688415 2024-12-07T00:51:28,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688415 2024-12-07T00:51:28,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688415) roll requested 2024-12-07T00:51:28,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688425 2024-12-07T00:51:28,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688425 2024-12-07T00:51:28,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688425) roll requested 2024-12-07T00:51:28,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688434 2024-12-07T00:51:28,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688434 2024-12-07T00:51:28,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688434) roll requested 2024-12-07T00:51:28,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688445 2024-12-07T00:51:28,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688445 2024-12-07T00:51:28,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688445) roll requested 2024-12-07T00:51:28,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688453 2024-12-07T00:51:28,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688453 2024-12-07T00:51:28,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688453) roll requested 2024-12-07T00:51:28,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688461 2024-12-07T00:51:28,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688461 2024-12-07T00:51:28,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688461) roll requested 2024-12-07T00:51:28,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688471 2024-12-07T00:51:28,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688471 2024-12-07T00:51:28,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688471) roll requested 2024-12-07T00:51:28,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688480 2024-12-07T00:51:28,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688480 2024-12-07T00:51:28,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688480) roll requested 2024-12-07T00:51:28,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688487 2024-12-07T00:51:28,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688487 2024-12-07T00:51:28,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688487) roll requested 2024-12-07T00:51:28,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688493 2024-12-07T00:51:28,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688493 2024-12-07T00:51:28,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688493) roll requested 2024-12-07T00:51:28,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688501 2024-12-07T00:51:28,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688501 2024-12-07T00:51:28,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688501) roll requested 2024-12-07T00:51:28,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688507 2024-12-07T00:51:28,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688507 2024-12-07T00:51:28,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688507) roll requested 2024-12-07T00:51:28,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688513 2024-12-07T00:51:28,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688513 2024-12-07T00:51:28,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688513) roll requested 2024-12-07T00:51:28,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688519 2024-12-07T00:51:28,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688519 2024-12-07T00:51:28,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688519) roll requested 2024-12-07T00:51:28,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688525 2024-12-07T00:51:28,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688525 2024-12-07T00:51:28,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688525) roll requested 2024-12-07T00:51:28,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688530 2024-12-07T00:51:28,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688530 2024-12-07T00:51:28,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688530) roll requested 2024-12-07T00:51:28,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688535 2024-12-07T00:51:28,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688535 2024-12-07T00:51:28,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688535) roll requested 2024-12-07T00:51:28,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688540 2024-12-07T00:51:28,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688540 2024-12-07T00:51:28,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688540) roll requested 2024-12-07T00:51:28,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688546 2024-12-07T00:51:28,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688546 2024-12-07T00:51:28,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688546) roll requested 2024-12-07T00:51:28,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688551 2024-12-07T00:51:28,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688551 2024-12-07T00:51:28,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688551) roll requested 2024-12-07T00:51:28,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688557 2024-12-07T00:51:28,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688557 2024-12-07T00:51:28,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688557) roll requested 2024-12-07T00:51:28,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688563 2024-12-07T00:51:28,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688563 2024-12-07T00:51:28,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688563) roll requested 2024-12-07T00:51:28,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688568 2024-12-07T00:51:28,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688568 2024-12-07T00:51:28,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688568) roll requested 2024-12-07T00:51:28,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688573 2024-12-07T00:51:28,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688573 2024-12-07T00:51:28,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688573) roll requested 2024-12-07T00:51:28,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688578 2024-12-07T00:51:28,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688578 2024-12-07T00:51:28,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688578) roll requested 2024-12-07T00:51:28,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688584 2024-12-07T00:51:28,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688584 2024-12-07T00:51:28,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688584) roll requested 2024-12-07T00:51:28,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688589 2024-12-07T00:51:28,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688589 2024-12-07T00:51:28,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688589) roll requested 2024-12-07T00:51:28,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688595 2024-12-07T00:51:28,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688595 2024-12-07T00:51:28,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688595) roll requested 2024-12-07T00:51:28,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688600 2024-12-07T00:51:28,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688600 2024-12-07T00:51:28,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688600) roll requested 2024-12-07T00:51:28,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688606 2024-12-07T00:51:28,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688606 2024-12-07T00:51:28,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688606) roll requested 2024-12-07T00:51:28,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688612 2024-12-07T00:51:28,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688612 2024-12-07T00:51:28,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688612) roll requested 2024-12-07T00:51:28,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688618 2024-12-07T00:51:28,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688618 2024-12-07T00:51:28,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688618) roll requested 2024-12-07T00:51:28,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688623 2024-12-07T00:51:28,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688623 2024-12-07T00:51:28,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688623) roll requested 2024-12-07T00:51:28,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688629 2024-12-07T00:51:28,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688629 2024-12-07T00:51:28,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688629) roll requested 2024-12-07T00:51:28,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688634 2024-12-07T00:51:28,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688634 2024-12-07T00:51:28,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688634) roll requested 2024-12-07T00:51:28,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688640 2024-12-07T00:51:28,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688640 2024-12-07T00:51:28,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688640) roll requested 2024-12-07T00:51:28,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688645 2024-12-07T00:51:28,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688645 2024-12-07T00:51:28,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688645) roll requested 2024-12-07T00:51:28,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688650 2024-12-07T00:51:28,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688650 2024-12-07T00:51:28,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688650) roll requested 2024-12-07T00:51:28,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688656 2024-12-07T00:51:28,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688656 2024-12-07T00:51:28,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688656) roll requested 2024-12-07T00:51:28,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688661 2024-12-07T00:51:28,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688661 2024-12-07T00:51:28,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688661) roll requested 2024-12-07T00:51:28,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688668 2024-12-07T00:51:28,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688668 2024-12-07T00:51:28,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688668) roll requested 2024-12-07T00:51:28,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688673 2024-12-07T00:51:28,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688673 2024-12-07T00:51:28,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688673) roll requested 2024-12-07T00:51:28,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688679 2024-12-07T00:51:28,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688679 2024-12-07T00:51:28,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688679) roll requested 2024-12-07T00:51:28,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688685 2024-12-07T00:51:28,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688685 2024-12-07T00:51:28,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688685) roll requested 2024-12-07T00:51:28,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688690 2024-12-07T00:51:28,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688690 2024-12-07T00:51:28,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688690) roll requested 2024-12-07T00:51:28,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688696 2024-12-07T00:51:28,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688696 2024-12-07T00:51:28,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688696) roll requested 2024-12-07T00:51:28,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688701 2024-12-07T00:51:28,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688701 2024-12-07T00:51:28,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688701) roll requested 2024-12-07T00:51:28,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688707 2024-12-07T00:51:28,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688707 2024-12-07T00:51:28,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688707) roll requested 2024-12-07T00:51:28,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688712 2024-12-07T00:51:28,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688712 2024-12-07T00:51:28,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688712) roll requested 2024-12-07T00:51:28,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688718 2024-12-07T00:51:28,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688718 2024-12-07T00:51:28,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688718) roll requested 2024-12-07T00:51:28,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688723 2024-12-07T00:51:28,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688723 2024-12-07T00:51:28,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688723) roll requested 2024-12-07T00:51:28,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688728 2024-12-07T00:51:28,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688728 2024-12-07T00:51:28,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688728) roll requested 2024-12-07T00:51:28,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688733 2024-12-07T00:51:28,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688733 2024-12-07T00:51:28,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688733) roll requested 2024-12-07T00:51:28,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688739 2024-12-07T00:51:28,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688739 2024-12-07T00:51:28,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688739) roll requested 2024-12-07T00:51:28,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688743 2024-12-07T00:51:28,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688743 2024-12-07T00:51:28,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688743) roll requested 2024-12-07T00:51:28,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688748 2024-12-07T00:51:28,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688748 2024-12-07T00:51:28,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688748) roll requested 2024-12-07T00:51:28,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688753 2024-12-07T00:51:28,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688753 2024-12-07T00:51:28,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688753) roll requested 2024-12-07T00:51:28,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688759 2024-12-07T00:51:28,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688759 2024-12-07T00:51:28,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688759) roll requested 2024-12-07T00:51:28,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688766 2024-12-07T00:51:28,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688766 2024-12-07T00:51:28,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688766) roll requested 2024-12-07T00:51:28,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688775 2024-12-07T00:51:28,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688775 2024-12-07T00:51:28,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688775) roll requested 2024-12-07T00:51:28,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688783 2024-12-07T00:51:28,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688783 2024-12-07T00:51:28,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688783) roll requested 2024-12-07T00:51:28,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688791 2024-12-07T00:51:28,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688791 2024-12-07T00:51:28,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688791) roll requested 2024-12-07T00:51:28,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688801 2024-12-07T00:51:28,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688801 2024-12-07T00:51:28,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688801) roll requested 2024-12-07T00:51:28,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688806 2024-12-07T00:51:28,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688806 2024-12-07T00:51:28,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688806) roll requested 2024-12-07T00:51:28,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688810 2024-12-07T00:51:28,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688810 2024-12-07T00:51:28,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688810) roll requested 2024-12-07T00:51:28,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688816 2024-12-07T00:51:28,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688816 2024-12-07T00:51:28,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688816) roll requested 2024-12-07T00:51:28,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688822 2024-12-07T00:51:28,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688822 2024-12-07T00:51:28,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688822) roll requested 2024-12-07T00:51:28,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688829 2024-12-07T00:51:28,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688829 2024-12-07T00:51:28,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688829) roll requested 2024-12-07T00:51:28,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688836 2024-12-07T00:51:28,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688836 2024-12-07T00:51:28,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688836) roll requested 2024-12-07T00:51:28,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688842 2024-12-07T00:51:28,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688842 2024-12-07T00:51:28,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688842) roll requested 2024-12-07T00:51:28,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688848 2024-12-07T00:51:28,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688848 2024-12-07T00:51:28,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688848) roll requested 2024-12-07T00:51:28,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688855 2024-12-07T00:51:28,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688855 2024-12-07T00:51:28,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688855) roll requested 2024-12-07T00:51:28,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688863 2024-12-07T00:51:28,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688863 2024-12-07T00:51:28,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688863) roll requested 2024-12-07T00:51:28,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688871 2024-12-07T00:51:28,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688871 2024-12-07T00:51:28,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688871) roll requested 2024-12-07T00:51:28,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688880 2024-12-07T00:51:28,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688880 2024-12-07T00:51:28,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688880) roll requested 2024-12-07T00:51:28,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688887 2024-12-07T00:51:28,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688887 2024-12-07T00:51:28,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688887) roll requested 2024-12-07T00:51:28,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688895 2024-12-07T00:51:28,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688895 2024-12-07T00:51:28,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688895) roll requested 2024-12-07T00:51:28,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688900 2024-12-07T00:51:28,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688900 2024-12-07T00:51:28,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688900) roll requested 2024-12-07T00:51:28,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688905 2024-12-07T00:51:28,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688905 2024-12-07T00:51:28,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688905) roll requested 2024-12-07T00:51:28,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688915 2024-12-07T00:51:28,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688915 2024-12-07T00:51:28,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688915) roll requested 2024-12-07T00:51:28,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688921 2024-12-07T00:51:28,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688921 2024-12-07T00:51:28,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688921) roll requested 2024-12-07T00:51:28,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688927 2024-12-07T00:51:28,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688927 2024-12-07T00:51:28,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688927) roll requested 2024-12-07T00:51:28,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688933 2024-12-07T00:51:28,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688933 2024-12-07T00:51:28,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688933) roll requested 2024-12-07T00:51:28,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688942 2024-12-07T00:51:28,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688942 2024-12-07T00:51:28,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688942) roll requested 2024-12-07T00:51:28,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688948 2024-12-07T00:51:28,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688948 2024-12-07T00:51:28,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688948) roll requested 2024-12-07T00:51:28,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688958 2024-12-07T00:51:28,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688958 2024-12-07T00:51:28,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688958) roll requested 2024-12-07T00:51:28,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688969 2024-12-07T00:51:28,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688969 2024-12-07T00:51:28,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688969) roll requested 2024-12-07T00:51:28,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688974 2024-12-07T00:51:28,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688974 2024-12-07T00:51:28,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688974) roll requested 2024-12-07T00:51:28,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688980 2024-12-07T00:51:28,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688980 2024-12-07T00:51:28,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688980) roll requested 2024-12-07T00:51:28,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688986 2024-12-07T00:51:28,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688986 2024-12-07T00:51:28,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688986) roll requested 2024-12-07T00:51:28,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688991 2024-12-07T00:51:28,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:28,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688991 2024-12-07T00:51:28,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:28,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:28,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688991) roll requested 2024-12-07T00:51:28,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532688996 2024-12-07T00:51:29,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688996 2024-12-07T00:51:29,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532688996) roll requested 2024-12-07T00:51:29,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689002 2024-12-07T00:51:29,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532688996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689002 2024-12-07T00:51:29,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689002) roll requested 2024-12-07T00:51:29,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689009 2024-12-07T00:51:29,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689009 2024-12-07T00:51:29,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689009) roll requested 2024-12-07T00:51:29,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689016 2024-12-07T00:51:29,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689016 2024-12-07T00:51:29,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689016) roll requested 2024-12-07T00:51:29,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689022 2024-12-07T00:51:29,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689022 2024-12-07T00:51:29,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689022) roll requested 2024-12-07T00:51:29,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689028 2024-12-07T00:51:29,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689028 2024-12-07T00:51:29,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689028) roll requested 2024-12-07T00:51:29,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689034 2024-12-07T00:51:29,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689034 2024-12-07T00:51:29,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689034) roll requested 2024-12-07T00:51:29,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689040 2024-12-07T00:51:29,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689040 2024-12-07T00:51:29,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689040) roll requested 2024-12-07T00:51:29,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689045 2024-12-07T00:51:29,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689045 2024-12-07T00:51:29,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689045) roll requested 2024-12-07T00:51:29,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689052 2024-12-07T00:51:29,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689052 2024-12-07T00:51:29,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689052) roll requested 2024-12-07T00:51:29,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689060 2024-12-07T00:51:29,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689060 2024-12-07T00:51:29,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689060) roll requested 2024-12-07T00:51:29,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689065 2024-12-07T00:51:29,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689065 2024-12-07T00:51:29,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689065) roll requested 2024-12-07T00:51:29,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689071 2024-12-07T00:51:29,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689071 2024-12-07T00:51:29,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689071) roll requested 2024-12-07T00:51:29,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689076 2024-12-07T00:51:29,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689076 2024-12-07T00:51:29,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689076) roll requested 2024-12-07T00:51:29,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689082 2024-12-07T00:51:29,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689082 2024-12-07T00:51:29,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689082) roll requested 2024-12-07T00:51:29,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689087 2024-12-07T00:51:29,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689087 2024-12-07T00:51:29,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689087) roll requested 2024-12-07T00:51:29,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689092 2024-12-07T00:51:29,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689092 2024-12-07T00:51:29,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689092) roll requested 2024-12-07T00:51:29,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689098 2024-12-07T00:51:29,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689098 2024-12-07T00:51:29,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689098) roll requested 2024-12-07T00:51:29,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689104 2024-12-07T00:51:29,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689104 2024-12-07T00:51:29,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689104) roll requested 2024-12-07T00:51:29,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689110 2024-12-07T00:51:29,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689110 2024-12-07T00:51:29,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689110) roll requested 2024-12-07T00:51:29,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689116 2024-12-07T00:51:29,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689116 2024-12-07T00:51:29,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689116) roll requested 2024-12-07T00:51:29,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689122 2024-12-07T00:51:29,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689122 2024-12-07T00:51:29,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689122) roll requested 2024-12-07T00:51:29,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689128 2024-12-07T00:51:29,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689128 2024-12-07T00:51:29,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689128) roll requested 2024-12-07T00:51:29,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689134 2024-12-07T00:51:29,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689134 2024-12-07T00:51:29,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689134) roll requested 2024-12-07T00:51:29,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689140 2024-12-07T00:51:29,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689140 2024-12-07T00:51:29,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689140) roll requested 2024-12-07T00:51:29,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689148 2024-12-07T00:51:29,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689148 2024-12-07T00:51:29,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689148) roll requested 2024-12-07T00:51:29,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689154 2024-12-07T00:51:29,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689154 2024-12-07T00:51:29,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689154) roll requested 2024-12-07T00:51:29,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689160 2024-12-07T00:51:29,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689160 2024-12-07T00:51:29,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689160) roll requested 2024-12-07T00:51:29,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689169 2024-12-07T00:51:29,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689169 2024-12-07T00:51:29,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689169) roll requested 2024-12-07T00:51:29,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689174 2024-12-07T00:51:29,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689174 2024-12-07T00:51:29,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689174) roll requested 2024-12-07T00:51:29,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689179 2024-12-07T00:51:29,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689179 2024-12-07T00:51:29,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689179) roll requested 2024-12-07T00:51:29,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689183 2024-12-07T00:51:29,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689183 2024-12-07T00:51:29,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689183) roll requested 2024-12-07T00:51:29,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689188 2024-12-07T00:51:29,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689188 2024-12-07T00:51:29,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689188) roll requested 2024-12-07T00:51:29,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689193 2024-12-07T00:51:29,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689193 2024-12-07T00:51:29,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689193) roll requested 2024-12-07T00:51:29,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689197 2024-12-07T00:51:29,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689197 2024-12-07T00:51:29,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689197) roll requested 2024-12-07T00:51:29,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689202 2024-12-07T00:51:29,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689202 2024-12-07T00:51:29,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689202) roll requested 2024-12-07T00:51:29,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689207 2024-12-07T00:51:29,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689207 2024-12-07T00:51:29,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689207) roll requested 2024-12-07T00:51:29,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689212 2024-12-07T00:51:29,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689212 2024-12-07T00:51:29,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689212) roll requested 2024-12-07T00:51:29,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689224 2024-12-07T00:51:29,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689224 2024-12-07T00:51:29,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689224) roll requested 2024-12-07T00:51:29,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689231 2024-12-07T00:51:29,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689231 2024-12-07T00:51:29,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689231) roll requested 2024-12-07T00:51:29,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689241 2024-12-07T00:51:29,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689241 2024-12-07T00:51:29,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689241) roll requested 2024-12-07T00:51:29,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689246 2024-12-07T00:51:29,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689246 2024-12-07T00:51:29,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689246) roll requested 2024-12-07T00:51:29,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689251 2024-12-07T00:51:29,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689251 2024-12-07T00:51:29,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689251) roll requested 2024-12-07T00:51:29,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689256 2024-12-07T00:51:29,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689256 2024-12-07T00:51:29,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689256) roll requested 2024-12-07T00:51:29,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689261 2024-12-07T00:51:29,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689261 2024-12-07T00:51:29,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689261) roll requested 2024-12-07T00:51:29,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689266 2024-12-07T00:51:29,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689266 2024-12-07T00:51:29,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689266) roll requested 2024-12-07T00:51:29,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689273 2024-12-07T00:51:29,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689273 2024-12-07T00:51:29,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689273) roll requested 2024-12-07T00:51:29,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689277 2024-12-07T00:51:29,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689277 2024-12-07T00:51:29,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689277) roll requested 2024-12-07T00:51:29,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689282 2024-12-07T00:51:29,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689282 2024-12-07T00:51:29,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689282) roll requested 2024-12-07T00:51:29,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689288 2024-12-07T00:51:29,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689288 2024-12-07T00:51:29,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689288) roll requested 2024-12-07T00:51:29,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689294 2024-12-07T00:51:29,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689294 2024-12-07T00:51:29,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689294) roll requested 2024-12-07T00:51:29,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689299 2024-12-07T00:51:29,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689299 2024-12-07T00:51:29,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689299) roll requested 2024-12-07T00:51:29,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689304 2024-12-07T00:51:29,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689304 2024-12-07T00:51:29,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689304) roll requested 2024-12-07T00:51:29,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689310 2024-12-07T00:51:29,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689310 2024-12-07T00:51:29,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689310) roll requested 2024-12-07T00:51:29,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689315 2024-12-07T00:51:29,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689315 2024-12-07T00:51:29,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689315) roll requested 2024-12-07T00:51:29,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689320 2024-12-07T00:51:29,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689320 2024-12-07T00:51:29,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689320) roll requested 2024-12-07T00:51:29,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689325 2024-12-07T00:51:29,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689325 2024-12-07T00:51:29,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689325) roll requested 2024-12-07T00:51:29,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689331 2024-12-07T00:51:29,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689331 2024-12-07T00:51:29,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689331) roll requested 2024-12-07T00:51:29,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689336 2024-12-07T00:51:29,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689336 2024-12-07T00:51:29,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689336) roll requested 2024-12-07T00:51:29,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689341 2024-12-07T00:51:29,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689341 2024-12-07T00:51:29,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689341) roll requested 2024-12-07T00:51:29,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689347 2024-12-07T00:51:29,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689347 2024-12-07T00:51:29,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689347) roll requested 2024-12-07T00:51:29,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689361 2024-12-07T00:51:29,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689361 2024-12-07T00:51:29,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689361) roll requested 2024-12-07T00:51:29,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689373 2024-12-07T00:51:29,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689373 2024-12-07T00:51:29,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689373) roll requested 2024-12-07T00:51:29,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689378 2024-12-07T00:51:29,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689378 2024-12-07T00:51:29,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689378) roll requested 2024-12-07T00:51:29,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689387 2024-12-07T00:51:29,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689387 2024-12-07T00:51:29,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689387) roll requested 2024-12-07T00:51:29,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689392 2024-12-07T00:51:29,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689392 2024-12-07T00:51:29,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689392) roll requested 2024-12-07T00:51:29,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689397 2024-12-07T00:51:29,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689397 2024-12-07T00:51:29,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689397) roll requested 2024-12-07T00:51:29,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689402 2024-12-07T00:51:29,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689402 2024-12-07T00:51:29,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689402) roll requested 2024-12-07T00:51:29,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689411 2024-12-07T00:51:29,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689411 2024-12-07T00:51:29,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689411) roll requested 2024-12-07T00:51:29,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689416 2024-12-07T00:51:29,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689416 2024-12-07T00:51:29,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689416) roll requested 2024-12-07T00:51:29,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689421 2024-12-07T00:51:29,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689421 2024-12-07T00:51:29,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689421) roll requested 2024-12-07T00:51:29,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689431 2024-12-07T00:51:29,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689431 2024-12-07T00:51:29,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689431) roll requested 2024-12-07T00:51:29,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689437 2024-12-07T00:51:29,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689437 2024-12-07T00:51:29,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689437) roll requested 2024-12-07T00:51:29,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689444 2024-12-07T00:51:29,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689444 2024-12-07T00:51:29,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689444) roll requested 2024-12-07T00:51:29,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689453 2024-12-07T00:51:29,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689453 2024-12-07T00:51:29,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689453) roll requested 2024-12-07T00:51:29,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689458 2024-12-07T00:51:29,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689458 2024-12-07T00:51:29,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689458) roll requested 2024-12-07T00:51:29,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689463 2024-12-07T00:51:29,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689463 2024-12-07T00:51:29,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689463) roll requested 2024-12-07T00:51:29,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689468 2024-12-07T00:51:29,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689468 2024-12-07T00:51:29,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689468) roll requested 2024-12-07T00:51:29,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689473 2024-12-07T00:51:29,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689473 2024-12-07T00:51:29,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689473) roll requested 2024-12-07T00:51:29,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689478 2024-12-07T00:51:29,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689478 2024-12-07T00:51:29,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689478) roll requested 2024-12-07T00:51:29,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689483 2024-12-07T00:51:29,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689483 2024-12-07T00:51:29,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689483) roll requested 2024-12-07T00:51:29,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689490 2024-12-07T00:51:29,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689490 2024-12-07T00:51:29,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689490) roll requested 2024-12-07T00:51:29,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689496 2024-12-07T00:51:29,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689496 2024-12-07T00:51:29,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689496) roll requested 2024-12-07T00:51:29,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689502 2024-12-07T00:51:29,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689502 2024-12-07T00:51:29,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689502) roll requested 2024-12-07T00:51:29,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689512 2024-12-07T00:51:29,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689512 2024-12-07T00:51:29,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689512) roll requested 2024-12-07T00:51:29,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689521 2024-12-07T00:51:29,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689521 2024-12-07T00:51:29,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689521) roll requested 2024-12-07T00:51:29,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689527 2024-12-07T00:51:29,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689527 2024-12-07T00:51:29,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689527) roll requested 2024-12-07T00:51:29,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689533 2024-12-07T00:51:29,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689533 2024-12-07T00:51:29,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689533) roll requested 2024-12-07T00:51:29,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689539 2024-12-07T00:51:29,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689539 2024-12-07T00:51:29,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689539) roll requested 2024-12-07T00:51:29,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689547 2024-12-07T00:51:29,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689547 2024-12-07T00:51:29,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689547) roll requested 2024-12-07T00:51:29,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689557 2024-12-07T00:51:29,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689557 2024-12-07T00:51:29,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689557) roll requested 2024-12-07T00:51:29,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689567 2024-12-07T00:51:29,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689567 2024-12-07T00:51:29,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689567) roll requested 2024-12-07T00:51:29,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689575 2024-12-07T00:51:29,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689575 2024-12-07T00:51:29,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689575) roll requested 2024-12-07T00:51:29,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689582 2024-12-07T00:51:29,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689582 2024-12-07T00:51:29,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689582) roll requested 2024-12-07T00:51:29,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689588 2024-12-07T00:51:29,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689588 2024-12-07T00:51:29,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689588) roll requested 2024-12-07T00:51:29,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689596 2024-12-07T00:51:29,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689596 2024-12-07T00:51:29,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689596) roll requested 2024-12-07T00:51:29,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689603 2024-12-07T00:51:29,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689603 2024-12-07T00:51:29,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689603) roll requested 2024-12-07T00:51:29,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689608 2024-12-07T00:51:29,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689608 2024-12-07T00:51:29,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689608) roll requested 2024-12-07T00:51:29,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689652 2024-12-07T00:51:29,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689652 2024-12-07T00:51:29,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689652) roll requested 2024-12-07T00:51:29,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689660 2024-12-07T00:51:29,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689660 2024-12-07T00:51:29,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689660) roll requested 2024-12-07T00:51:29,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689666 2024-12-07T00:51:29,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689666 2024-12-07T00:51:29,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689666) roll requested 2024-12-07T00:51:29,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689671 2024-12-07T00:51:29,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689671 2024-12-07T00:51:29,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689671) roll requested 2024-12-07T00:51:29,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689676 2024-12-07T00:51:29,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689676 2024-12-07T00:51:29,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689676) roll requested 2024-12-07T00:51:29,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689681 2024-12-07T00:51:29,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689681 2024-12-07T00:51:29,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689681) roll requested 2024-12-07T00:51:29,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689686 2024-12-07T00:51:29,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689686 2024-12-07T00:51:29,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689686) roll requested 2024-12-07T00:51:29,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689691 2024-12-07T00:51:29,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689691 2024-12-07T00:51:29,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689691) roll requested 2024-12-07T00:51:29,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689696 2024-12-07T00:51:29,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689696 2024-12-07T00:51:29,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689696) roll requested 2024-12-07T00:51:29,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689702 2024-12-07T00:51:29,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689702 2024-12-07T00:51:29,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689702) roll requested 2024-12-07T00:51:29,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689707 2024-12-07T00:51:29,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689707 2024-12-07T00:51:29,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689707) roll requested 2024-12-07T00:51:29,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689713 2024-12-07T00:51:29,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689713 2024-12-07T00:51:29,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689713) roll requested 2024-12-07T00:51:29,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689719 2024-12-07T00:51:29,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689719 2024-12-07T00:51:29,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689719) roll requested 2024-12-07T00:51:29,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689724 2024-12-07T00:51:29,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689724 2024-12-07T00:51:29,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689724) roll requested 2024-12-07T00:51:29,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689730 2024-12-07T00:51:29,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689730 2024-12-07T00:51:29,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689730) roll requested 2024-12-07T00:51:29,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689735 2024-12-07T00:51:29,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689735 2024-12-07T00:51:29,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689735) roll requested 2024-12-07T00:51:29,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689741 2024-12-07T00:51:29,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689741 2024-12-07T00:51:29,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689741) roll requested 2024-12-07T00:51:29,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689746 2024-12-07T00:51:29,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689746 2024-12-07T00:51:29,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689746) roll requested 2024-12-07T00:51:29,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689751 2024-12-07T00:51:29,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689751 2024-12-07T00:51:29,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689751) roll requested 2024-12-07T00:51:29,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689756 2024-12-07T00:51:29,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689756 2024-12-07T00:51:29,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689756) roll requested 2024-12-07T00:51:29,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689762 2024-12-07T00:51:29,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689762 2024-12-07T00:51:29,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689762) roll requested 2024-12-07T00:51:29,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689767 2024-12-07T00:51:29,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689767 2024-12-07T00:51:29,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689767) roll requested 2024-12-07T00:51:29,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689772 2024-12-07T00:51:29,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689772 2024-12-07T00:51:29,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689772) roll requested 2024-12-07T00:51:29,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689777 2024-12-07T00:51:29,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689777 2024-12-07T00:51:29,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689777) roll requested 2024-12-07T00:51:29,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689783 2024-12-07T00:51:29,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689783 2024-12-07T00:51:29,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689783) roll requested 2024-12-07T00:51:29,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689789 2024-12-07T00:51:29,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689789 2024-12-07T00:51:29,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689789) roll requested 2024-12-07T00:51:29,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689795 2024-12-07T00:51:29,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689795 2024-12-07T00:51:29,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689795) roll requested 2024-12-07T00:51:29,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689801 2024-12-07T00:51:29,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689801 2024-12-07T00:51:29,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689801) roll requested 2024-12-07T00:51:29,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689806 2024-12-07T00:51:29,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689806 2024-12-07T00:51:29,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689806) roll requested 2024-12-07T00:51:29,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689811 2024-12-07T00:51:29,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689811 2024-12-07T00:51:29,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689811) roll requested 2024-12-07T00:51:29,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689816 2024-12-07T00:51:29,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689816 2024-12-07T00:51:29,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689816) roll requested 2024-12-07T00:51:29,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689823 2024-12-07T00:51:29,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689823 2024-12-07T00:51:29,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689823) roll requested 2024-12-07T00:51:29,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689833 2024-12-07T00:51:29,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689833 2024-12-07T00:51:29,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689833) roll requested 2024-12-07T00:51:29,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689843 2024-12-07T00:51:29,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689843 2024-12-07T00:51:29,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689843) roll requested 2024-12-07T00:51:29,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689847 2024-12-07T00:51:29,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689847 2024-12-07T00:51:29,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689847) roll requested 2024-12-07T00:51:29,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689853 2024-12-07T00:51:29,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689853 2024-12-07T00:51:29,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689853) roll requested 2024-12-07T00:51:29,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689861 2024-12-07T00:51:29,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689861 2024-12-07T00:51:29,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689861) roll requested 2024-12-07T00:51:29,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689867 2024-12-07T00:51:29,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689867 2024-12-07T00:51:29,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689867) roll requested 2024-12-07T00:51:29,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689872 2024-12-07T00:51:29,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689872 2024-12-07T00:51:29,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689872) roll requested 2024-12-07T00:51:29,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689878 2024-12-07T00:51:29,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689878 2024-12-07T00:51:29,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689878) roll requested 2024-12-07T00:51:29,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689884 2024-12-07T00:51:29,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689884 2024-12-07T00:51:29,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689884) roll requested 2024-12-07T00:51:29,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689890 2024-12-07T00:51:29,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689890 2024-12-07T00:51:29,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689890) roll requested 2024-12-07T00:51:29,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689896 2024-12-07T00:51:29,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689896 2024-12-07T00:51:29,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689896) roll requested 2024-12-07T00:51:29,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689903 2024-12-07T00:51:29,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689903 2024-12-07T00:51:29,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689903) roll requested 2024-12-07T00:51:29,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689911 2024-12-07T00:51:29,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689911 2024-12-07T00:51:29,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689911) roll requested 2024-12-07T00:51:29,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689919 2024-12-07T00:51:29,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689919 2024-12-07T00:51:29,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689919) roll requested 2024-12-07T00:51:29,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689927 2024-12-07T00:51:29,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689927 2024-12-07T00:51:29,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689927) roll requested 2024-12-07T00:51:29,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689936 2024-12-07T00:51:29,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689936 2024-12-07T00:51:29,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689936) roll requested 2024-12-07T00:51:29,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689945 2024-12-07T00:51:29,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689945 2024-12-07T00:51:29,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689945) roll requested 2024-12-07T00:51:29,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689954 2024-12-07T00:51:29,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689954 2024-12-07T00:51:29,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689954) roll requested 2024-12-07T00:51:29,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689962 2024-12-07T00:51:29,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689962 2024-12-07T00:51:29,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689962) roll requested 2024-12-07T00:51:29,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689969 2024-12-07T00:51:29,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689969 2024-12-07T00:51:29,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689969) roll requested 2024-12-07T00:51:29,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689977 2024-12-07T00:51:29,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689977 2024-12-07T00:51:29,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689977) roll requested 2024-12-07T00:51:29,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689987 2024-12-07T00:51:29,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:29,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689987 2024-12-07T00:51:29,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:29,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:29,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689987) roll requested 2024-12-07T00:51:29,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532689997 2024-12-07T00:51:30,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689997 2024-12-07T00:51:30,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532689997) roll requested 2024-12-07T00:51:30,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690005 2024-12-07T00:51:30,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532689997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690005 2024-12-07T00:51:30,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690005) roll requested 2024-12-07T00:51:30,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690014 2024-12-07T00:51:30,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690014 2024-12-07T00:51:30,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690014) roll requested 2024-12-07T00:51:30,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690021 2024-12-07T00:51:30,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690021 2024-12-07T00:51:30,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690021) roll requested 2024-12-07T00:51:30,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690028 2024-12-07T00:51:30,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690028 2024-12-07T00:51:30,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690028) roll requested 2024-12-07T00:51:30,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690035 2024-12-07T00:51:30,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690035 2024-12-07T00:51:30,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690035) roll requested 2024-12-07T00:51:30,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690044 2024-12-07T00:51:30,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690044 2024-12-07T00:51:30,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690044) roll requested 2024-12-07T00:51:30,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690054 2024-12-07T00:51:30,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690054 2024-12-07T00:51:30,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690054) roll requested 2024-12-07T00:51:30,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690063 2024-12-07T00:51:30,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690063 2024-12-07T00:51:30,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690063) roll requested 2024-12-07T00:51:30,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690072 2024-12-07T00:51:30,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690072 2024-12-07T00:51:30,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690072) roll requested 2024-12-07T00:51:30,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690079 2024-12-07T00:51:30,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690079 2024-12-07T00:51:30,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690079) roll requested 2024-12-07T00:51:30,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690086 2024-12-07T00:51:30,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690086 2024-12-07T00:51:30,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690086) roll requested 2024-12-07T00:51:30,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690093 2024-12-07T00:51:30,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690093 2024-12-07T00:51:30,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690093) roll requested 2024-12-07T00:51:30,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690103 2024-12-07T00:51:30,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690103 2024-12-07T00:51:30,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690103) roll requested 2024-12-07T00:51:30,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690112 2024-12-07T00:51:30,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690112 2024-12-07T00:51:30,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690112) roll requested 2024-12-07T00:51:30,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690121 2024-12-07T00:51:30,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690121 2024-12-07T00:51:30,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690121) roll requested 2024-12-07T00:51:30,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690128 2024-12-07T00:51:30,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690128 2024-12-07T00:51:30,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690128) roll requested 2024-12-07T00:51:30,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690135 2024-12-07T00:51:30,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690135 2024-12-07T00:51:30,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690135) roll requested 2024-12-07T00:51:30,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690143 2024-12-07T00:51:30,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690143 2024-12-07T00:51:30,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690143) roll requested 2024-12-07T00:51:30,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690151 2024-12-07T00:51:30,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690151 2024-12-07T00:51:30,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690151) roll requested 2024-12-07T00:51:30,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690160 2024-12-07T00:51:30,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690160 2024-12-07T00:51:30,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690160) roll requested 2024-12-07T00:51:30,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690166 2024-12-07T00:51:30,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690166 2024-12-07T00:51:30,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690166) roll requested 2024-12-07T00:51:30,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690173 2024-12-07T00:51:30,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690173 2024-12-07T00:51:30,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690173) roll requested 2024-12-07T00:51:30,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690180 2024-12-07T00:51:30,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690180 2024-12-07T00:51:30,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690180) roll requested 2024-12-07T00:51:30,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690188 2024-12-07T00:51:30,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690188 2024-12-07T00:51:30,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690188) roll requested 2024-12-07T00:51:30,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690198 2024-12-07T00:51:30,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690198 2024-12-07T00:51:30,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690198) roll requested 2024-12-07T00:51:30,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690206 2024-12-07T00:51:30,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690206 2024-12-07T00:51:30,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690206) roll requested 2024-12-07T00:51:30,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690211 2024-12-07T00:51:30,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690211 2024-12-07T00:51:30,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690211) roll requested 2024-12-07T00:51:30,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690218 2024-12-07T00:51:30,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690218 2024-12-07T00:51:30,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690218) roll requested 2024-12-07T00:51:30,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690224 2024-12-07T00:51:30,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690224 2024-12-07T00:51:30,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690224) roll requested 2024-12-07T00:51:30,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690232 2024-12-07T00:51:30,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690232 2024-12-07T00:51:30,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690232) roll requested 2024-12-07T00:51:30,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690240 2024-12-07T00:51:30,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690240 2024-12-07T00:51:30,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690240) roll requested 2024-12-07T00:51:30,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690247 2024-12-07T00:51:30,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690247 2024-12-07T00:51:30,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690247) roll requested 2024-12-07T00:51:30,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690255 2024-12-07T00:51:30,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690255 2024-12-07T00:51:30,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690255) roll requested 2024-12-07T00:51:30,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690261 2024-12-07T00:51:30,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690261 2024-12-07T00:51:30,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690261) roll requested 2024-12-07T00:51:30,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690268 2024-12-07T00:51:30,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690268 2024-12-07T00:51:30,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690268) roll requested 2024-12-07T00:51:30,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690276 2024-12-07T00:51:30,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690276 2024-12-07T00:51:30,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690276) roll requested 2024-12-07T00:51:30,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690282 2024-12-07T00:51:30,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690282 2024-12-07T00:51:30,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690282) roll requested 2024-12-07T00:51:30,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690288 2024-12-07T00:51:30,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690288 2024-12-07T00:51:30,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690288) roll requested 2024-12-07T00:51:30,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690295 2024-12-07T00:51:30,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690295 2024-12-07T00:51:30,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690295) roll requested 2024-12-07T00:51:30,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690303 2024-12-07T00:51:30,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690303 2024-12-07T00:51:30,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690303) roll requested 2024-12-07T00:51:30,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690313 2024-12-07T00:51:30,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690313 2024-12-07T00:51:30,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690313) roll requested 2024-12-07T00:51:30,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690322 2024-12-07T00:51:30,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690322 2024-12-07T00:51:30,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690322) roll requested 2024-12-07T00:51:30,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690330 2024-12-07T00:51:30,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690330 2024-12-07T00:51:30,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690330) roll requested 2024-12-07T00:51:30,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690337 2024-12-07T00:51:30,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690337 2024-12-07T00:51:30,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690337) roll requested 2024-12-07T00:51:30,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690344 2024-12-07T00:51:30,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690344 2024-12-07T00:51:30,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690344) roll requested 2024-12-07T00:51:30,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690353 2024-12-07T00:51:30,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690353 2024-12-07T00:51:30,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690353) roll requested 2024-12-07T00:51:30,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690361 2024-12-07T00:51:30,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690361 2024-12-07T00:51:30,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690361) roll requested 2024-12-07T00:51:30,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690368 2024-12-07T00:51:30,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690368 2024-12-07T00:51:30,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690368) roll requested 2024-12-07T00:51:30,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690375 2024-12-07T00:51:30,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690375 2024-12-07T00:51:30,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690375) roll requested 2024-12-07T00:51:30,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690383 2024-12-07T00:51:30,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690383 2024-12-07T00:51:30,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690383) roll requested 2024-12-07T00:51:30,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690389 2024-12-07T00:51:30,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690389 2024-12-07T00:51:30,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690389) roll requested 2024-12-07T00:51:30,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690397 2024-12-07T00:51:30,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690397 2024-12-07T00:51:30,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690397) roll requested 2024-12-07T00:51:30,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690405 2024-12-07T00:51:30,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690405 2024-12-07T00:51:30,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690405) roll requested 2024-12-07T00:51:30,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690412 2024-12-07T00:51:30,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690412 2024-12-07T00:51:30,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690412) roll requested 2024-12-07T00:51:30,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690419 2024-12-07T00:51:30,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690419 2024-12-07T00:51:30,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690419) roll requested 2024-12-07T00:51:30,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690426 2024-12-07T00:51:30,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690426 2024-12-07T00:51:30,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690426) roll requested 2024-12-07T00:51:30,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690434 2024-12-07T00:51:30,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690434 2024-12-07T00:51:30,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690434) roll requested 2024-12-07T00:51:30,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690443 2024-12-07T00:51:30,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690443 2024-12-07T00:51:30,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690443) roll requested 2024-12-07T00:51:30,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690452 2024-12-07T00:51:30,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690452 2024-12-07T00:51:30,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690452) roll requested 2024-12-07T00:51:30,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690461 2024-12-07T00:51:30,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690461 2024-12-07T00:51:30,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690461) roll requested 2024-12-07T00:51:30,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690472 2024-12-07T00:51:30,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690472 2024-12-07T00:51:30,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690472) roll requested 2024-12-07T00:51:30,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690479 2024-12-07T00:51:30,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690479 2024-12-07T00:51:30,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690479) roll requested 2024-12-07T00:51:30,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690484 2024-12-07T00:51:30,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690484 2024-12-07T00:51:30,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690484) roll requested 2024-12-07T00:51:30,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690490 2024-12-07T00:51:30,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690490 2024-12-07T00:51:30,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690490) roll requested 2024-12-07T00:51:30,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690498 2024-12-07T00:51:30,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690498 2024-12-07T00:51:30,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690498) roll requested 2024-12-07T00:51:30,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690504 2024-12-07T00:51:30,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690504 2024-12-07T00:51:30,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690504) roll requested 2024-12-07T00:51:30,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690512 2024-12-07T00:51:30,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690512 2024-12-07T00:51:30,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690512) roll requested 2024-12-07T00:51:30,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690524 2024-12-07T00:51:30,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690524 2024-12-07T00:51:30,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690524) roll requested 2024-12-07T00:51:30,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690536 2024-12-07T00:51:30,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690536 2024-12-07T00:51:30,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690536) roll requested 2024-12-07T00:51:30,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690542 2024-12-07T00:51:30,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690542 2024-12-07T00:51:30,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690542) roll requested 2024-12-07T00:51:30,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690547 2024-12-07T00:51:30,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690547 2024-12-07T00:51:30,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690547) roll requested 2024-12-07T00:51:30,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690553 2024-12-07T00:51:30,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690553 2024-12-07T00:51:30,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690553) roll requested 2024-12-07T00:51:30,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690559 2024-12-07T00:51:30,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690559 2024-12-07T00:51:30,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690559) roll requested 2024-12-07T00:51:30,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690567 2024-12-07T00:51:30,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690567 2024-12-07T00:51:30,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690567) roll requested 2024-12-07T00:51:30,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690575 2024-12-07T00:51:30,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690575 2024-12-07T00:51:30,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690575) roll requested 2024-12-07T00:51:30,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690583 2024-12-07T00:51:30,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690583 2024-12-07T00:51:30,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690583) roll requested 2024-12-07T00:51:30,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690593 2024-12-07T00:51:30,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690593 2024-12-07T00:51:30,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690593) roll requested 2024-12-07T00:51:30,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690603 2024-12-07T00:51:30,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690603 2024-12-07T00:51:30,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690603) roll requested 2024-12-07T00:51:30,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690610 2024-12-07T00:51:30,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690610 2024-12-07T00:51:30,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690610) roll requested 2024-12-07T00:51:30,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690616 2024-12-07T00:51:30,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690616 2024-12-07T00:51:30,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690616) roll requested 2024-12-07T00:51:30,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690624 2024-12-07T00:51:30,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690624 2024-12-07T00:51:30,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690624) roll requested 2024-12-07T00:51:30,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690631 2024-12-07T00:51:30,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690631 2024-12-07T00:51:30,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690631) roll requested 2024-12-07T00:51:30,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690639 2024-12-07T00:51:30,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690639 2024-12-07T00:51:30,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690639) roll requested 2024-12-07T00:51:30,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690646 2024-12-07T00:51:30,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690646 2024-12-07T00:51:30,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690646) roll requested 2024-12-07T00:51:30,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690655 2024-12-07T00:51:30,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690655 2024-12-07T00:51:30,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690655) roll requested 2024-12-07T00:51:30,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690664 2024-12-07T00:51:30,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690664 2024-12-07T00:51:30,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690664) roll requested 2024-12-07T00:51:30,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690672 2024-12-07T00:51:30,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690672 2024-12-07T00:51:30,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690672) roll requested 2024-12-07T00:51:30,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690680 2024-12-07T00:51:30,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690680 2024-12-07T00:51:30,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690680) roll requested 2024-12-07T00:51:30,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690685 2024-12-07T00:51:30,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690685 2024-12-07T00:51:30,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690685) roll requested 2024-12-07T00:51:30,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690690 2024-12-07T00:51:30,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690690 2024-12-07T00:51:30,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690690) roll requested 2024-12-07T00:51:30,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690697 2024-12-07T00:51:30,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690697 2024-12-07T00:51:30,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690697) roll requested 2024-12-07T00:51:30,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690704 2024-12-07T00:51:30,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690704 2024-12-07T00:51:30,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690704) roll requested 2024-12-07T00:51:30,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690711 2024-12-07T00:51:30,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690711 2024-12-07T00:51:30,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690711) roll requested 2024-12-07T00:51:30,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690716 2024-12-07T00:51:30,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690716 2024-12-07T00:51:30,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690716) roll requested 2024-12-07T00:51:30,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690721 2024-12-07T00:51:30,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690721 2024-12-07T00:51:30,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690721) roll requested 2024-12-07T00:51:30,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690727 2024-12-07T00:51:30,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690727 2024-12-07T00:51:30,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690727) roll requested 2024-12-07T00:51:30,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690734 2024-12-07T00:51:30,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690734 2024-12-07T00:51:30,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690734) roll requested 2024-12-07T00:51:30,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690742 2024-12-07T00:51:30,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690742 2024-12-07T00:51:30,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690742) roll requested 2024-12-07T00:51:30,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690751 2024-12-07T00:51:30,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690751 2024-12-07T00:51:30,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690751) roll requested 2024-12-07T00:51:30,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690759 2024-12-07T00:51:30,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690759 2024-12-07T00:51:30,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690759) roll requested 2024-12-07T00:51:30,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690768 2024-12-07T00:51:30,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690768 2024-12-07T00:51:30,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690768) roll requested 2024-12-07T00:51:30,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690778 2024-12-07T00:51:30,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690778 2024-12-07T00:51:30,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690778) roll requested 2024-12-07T00:51:30,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690789 2024-12-07T00:51:30,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690789 2024-12-07T00:51:30,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690789) roll requested 2024-12-07T00:51:30,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690798 2024-12-07T00:51:30,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690798 2024-12-07T00:51:30,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690798) roll requested 2024-12-07T00:51:30,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690804 2024-12-07T00:51:30,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690804 2024-12-07T00:51:30,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690804) roll requested 2024-12-07T00:51:30,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690810 2024-12-07T00:51:30,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690810 2024-12-07T00:51:30,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690810) roll requested 2024-12-07T00:51:30,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690817 2024-12-07T00:51:30,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690817 2024-12-07T00:51:30,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690817) roll requested 2024-12-07T00:51:30,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690824 2024-12-07T00:51:30,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690824 2024-12-07T00:51:30,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690824) roll requested 2024-12-07T00:51:30,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690831 2024-12-07T00:51:30,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690831 2024-12-07T00:51:30,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690831) roll requested 2024-12-07T00:51:30,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690839 2024-12-07T00:51:30,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690839 2024-12-07T00:51:30,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690839) roll requested 2024-12-07T00:51:30,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690847 2024-12-07T00:51:30,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690847 2024-12-07T00:51:30,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690847) roll requested 2024-12-07T00:51:30,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690855 2024-12-07T00:51:30,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690855 2024-12-07T00:51:30,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690855) roll requested 2024-12-07T00:51:30,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690864 2024-12-07T00:51:30,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690864 2024-12-07T00:51:30,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690864) roll requested 2024-12-07T00:51:30,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690871 2024-12-07T00:51:30,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690871 2024-12-07T00:51:30,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690871) roll requested 2024-12-07T00:51:30,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690877 2024-12-07T00:51:30,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690877 2024-12-07T00:51:30,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690877) roll requested 2024-12-07T00:51:30,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690883 2024-12-07T00:51:30,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690883 2024-12-07T00:51:30,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690883) roll requested 2024-12-07T00:51:30,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690890 2024-12-07T00:51:30,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690890 2024-12-07T00:51:30,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690890) roll requested 2024-12-07T00:51:30,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690895 2024-12-07T00:51:30,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690895 2024-12-07T00:51:30,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690895) roll requested 2024-12-07T00:51:30,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690900 2024-12-07T00:51:30,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690900 2024-12-07T00:51:30,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690900) roll requested 2024-12-07T00:51:30,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690908 2024-12-07T00:51:30,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690908 2024-12-07T00:51:30,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690908) roll requested 2024-12-07T00:51:30,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690914 2024-12-07T00:51:30,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690914 2024-12-07T00:51:30,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690914) roll requested 2024-12-07T00:51:30,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690921 2024-12-07T00:51:30,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690921 2024-12-07T00:51:30,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690921) roll requested 2024-12-07T00:51:30,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690928 2024-12-07T00:51:30,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690928 2024-12-07T00:51:30,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690928) roll requested 2024-12-07T00:51:30,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690938 2024-12-07T00:51:30,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690938 2024-12-07T00:51:30,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690938) roll requested 2024-12-07T00:51:30,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690946 2024-12-07T00:51:30,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690946 2024-12-07T00:51:30,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690946) roll requested 2024-12-07T00:51:30,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690954 2024-12-07T00:51:30,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690954 2024-12-07T00:51:30,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690954) roll requested 2024-12-07T00:51:30,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690964 2024-12-07T00:51:30,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690964 2024-12-07T00:51:30,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690964) roll requested 2024-12-07T00:51:30,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690972 2024-12-07T00:51:30,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690972 2024-12-07T00:51:30,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690972) roll requested 2024-12-07T00:51:30,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690980 2024-12-07T00:51:30,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690980 2024-12-07T00:51:30,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690980) roll requested 2024-12-07T00:51:30,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690988 2024-12-07T00:51:30,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:30,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690988 2024-12-07T00:51:30,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:30,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:30,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690988) roll requested 2024-12-07T00:51:30,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532690995 2024-12-07T00:51:31,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690995 2024-12-07T00:51:31,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532690995) roll requested 2024-12-07T00:51:31,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691006 2024-12-07T00:51:31,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532690995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691006 2024-12-07T00:51:31,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691006) roll requested 2024-12-07T00:51:31,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691014 2024-12-07T00:51:31,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691014 2024-12-07T00:51:31,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691014) roll requested 2024-12-07T00:51:31,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691021 2024-12-07T00:51:31,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691021 2024-12-07T00:51:31,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691021) roll requested 2024-12-07T00:51:31,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691035 2024-12-07T00:51:31,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691035 2024-12-07T00:51:31,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691035) roll requested 2024-12-07T00:51:31,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691041 2024-12-07T00:51:31,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691041 2024-12-07T00:51:31,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691041) roll requested 2024-12-07T00:51:31,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691048 2024-12-07T00:51:31,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691048 2024-12-07T00:51:31,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691048) roll requested 2024-12-07T00:51:31,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691054 2024-12-07T00:51:31,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691054 2024-12-07T00:51:31,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691054) roll requested 2024-12-07T00:51:31,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691061 2024-12-07T00:51:31,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691061 2024-12-07T00:51:31,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691061) roll requested 2024-12-07T00:51:31,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691068 2024-12-07T00:51:31,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691068 2024-12-07T00:51:31,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691068) roll requested 2024-12-07T00:51:31,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691075 2024-12-07T00:51:31,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691075 2024-12-07T00:51:31,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691075) roll requested 2024-12-07T00:51:31,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691083 2024-12-07T00:51:31,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691083 2024-12-07T00:51:31,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691083) roll requested 2024-12-07T00:51:31,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691091 2024-12-07T00:51:31,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691091 2024-12-07T00:51:31,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691091) roll requested 2024-12-07T00:51:31,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691099 2024-12-07T00:51:31,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691099 2024-12-07T00:51:31,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691099) roll requested 2024-12-07T00:51:31,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691109 2024-12-07T00:51:31,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691109 2024-12-07T00:51:31,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691109) roll requested 2024-12-07T00:51:31,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691118 2024-12-07T00:51:31,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691118 2024-12-07T00:51:31,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691118) roll requested 2024-12-07T00:51:31,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691125 2024-12-07T00:51:31,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691125 2024-12-07T00:51:31,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691125) roll requested 2024-12-07T00:51:31,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691133 2024-12-07T00:51:31,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691133 2024-12-07T00:51:31,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691133) roll requested 2024-12-07T00:51:31,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691141 2024-12-07T00:51:31,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691141 2024-12-07T00:51:31,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691141) roll requested 2024-12-07T00:51:31,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691148 2024-12-07T00:51:31,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691148 2024-12-07T00:51:31,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691148) roll requested 2024-12-07T00:51:31,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691154 2024-12-07T00:51:31,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691154 2024-12-07T00:51:31,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691154) roll requested 2024-12-07T00:51:31,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691161 2024-12-07T00:51:31,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691161 2024-12-07T00:51:31,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691161) roll requested 2024-12-07T00:51:31,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691168 2024-12-07T00:51:31,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691168 2024-12-07T00:51:31,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691168) roll requested 2024-12-07T00:51:31,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691175 2024-12-07T00:51:31,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691175 2024-12-07T00:51:31,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691175) roll requested 2024-12-07T00:51:31,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691184 2024-12-07T00:51:31,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691184 2024-12-07T00:51:31,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691184) roll requested 2024-12-07T00:51:31,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691191 2024-12-07T00:51:31,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691191 2024-12-07T00:51:31,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691191) roll requested 2024-12-07T00:51:31,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691197 2024-12-07T00:51:31,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691197 2024-12-07T00:51:31,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691197) roll requested 2024-12-07T00:51:31,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691203 2024-12-07T00:51:31,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691203 2024-12-07T00:51:31,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691203) roll requested 2024-12-07T00:51:31,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691210 2024-12-07T00:51:31,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691210 2024-12-07T00:51:31,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691210) roll requested 2024-12-07T00:51:31,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691218 2024-12-07T00:51:31,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691218 2024-12-07T00:51:31,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691218) roll requested 2024-12-07T00:51:31,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691229 2024-12-07T00:51:31,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691229 2024-12-07T00:51:31,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691229) roll requested 2024-12-07T00:51:31,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691239 2024-12-07T00:51:31,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691239 2024-12-07T00:51:31,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691239) roll requested 2024-12-07T00:51:31,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691248 2024-12-07T00:51:31,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691248 2024-12-07T00:51:31,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691248) roll requested 2024-12-07T00:51:31,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691257 2024-12-07T00:51:31,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691257 2024-12-07T00:51:31,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691257) roll requested 2024-12-07T00:51:31,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691263 2024-12-07T00:51:31,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691263 2024-12-07T00:51:31,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691263) roll requested 2024-12-07T00:51:31,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691269 2024-12-07T00:51:31,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691269 2024-12-07T00:51:31,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691269) roll requested 2024-12-07T00:51:31,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691276 2024-12-07T00:51:31,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691276 2024-12-07T00:51:31,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691276) roll requested 2024-12-07T00:51:31,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691284 2024-12-07T00:51:31,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691284 2024-12-07T00:51:31,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691284) roll requested 2024-12-07T00:51:31,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691293 2024-12-07T00:51:31,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691293 2024-12-07T00:51:31,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691293) roll requested 2024-12-07T00:51:31,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691302 2024-12-07T00:51:31,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691302 2024-12-07T00:51:31,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691302) roll requested 2024-12-07T00:51:31,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691311 2024-12-07T00:51:31,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691311 2024-12-07T00:51:31,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691311) roll requested 2024-12-07T00:51:31,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691319 2024-12-07T00:51:31,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691319 2024-12-07T00:51:31,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691319) roll requested 2024-12-07T00:51:31,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691329 2024-12-07T00:51:31,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691329 2024-12-07T00:51:31,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691329) roll requested 2024-12-07T00:51:31,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691336 2024-12-07T00:51:31,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691336 2024-12-07T00:51:31,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691336) roll requested 2024-12-07T00:51:31,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691343 2024-12-07T00:51:31,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691343 2024-12-07T00:51:31,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691343) roll requested 2024-12-07T00:51:31,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691350 2024-12-07T00:51:31,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691350 2024-12-07T00:51:31,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691350) roll requested 2024-12-07T00:51:31,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691359 2024-12-07T00:51:31,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691359 2024-12-07T00:51:31,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691359) roll requested 2024-12-07T00:51:31,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691366 2024-12-07T00:51:31,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691366 2024-12-07T00:51:31,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691366) roll requested 2024-12-07T00:51:31,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691373 2024-12-07T00:51:31,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691373 2024-12-07T00:51:31,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691373) roll requested 2024-12-07T00:51:31,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691382 2024-12-07T00:51:31,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691382 2024-12-07T00:51:31,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691382) roll requested 2024-12-07T00:51:31,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691389 2024-12-07T00:51:31,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691389 2024-12-07T00:51:31,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691389) roll requested 2024-12-07T00:51:31,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691394 2024-12-07T00:51:31,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691394 2024-12-07T00:51:31,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691394) roll requested 2024-12-07T00:51:31,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691400 2024-12-07T00:51:31,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691400 2024-12-07T00:51:31,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691400) roll requested 2024-12-07T00:51:31,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691407 2024-12-07T00:51:31,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691407 2024-12-07T00:51:31,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691407) roll requested 2024-12-07T00:51:31,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691415 2024-12-07T00:51:31,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691415 2024-12-07T00:51:31,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691415) roll requested 2024-12-07T00:51:31,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691423 2024-12-07T00:51:31,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691423 2024-12-07T00:51:31,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691423) roll requested 2024-12-07T00:51:31,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691430 2024-12-07T00:51:31,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691430 2024-12-07T00:51:31,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691430) roll requested 2024-12-07T00:51:31,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691438 2024-12-07T00:51:31,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691438 2024-12-07T00:51:31,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691438) roll requested 2024-12-07T00:51:31,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691445 2024-12-07T00:51:31,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691445 2024-12-07T00:51:31,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691445) roll requested 2024-12-07T00:51:31,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691452 2024-12-07T00:51:31,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691452 2024-12-07T00:51:31,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691452) roll requested 2024-12-07T00:51:31,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691460 2024-12-07T00:51:31,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691460 2024-12-07T00:51:31,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691460) roll requested 2024-12-07T00:51:31,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691467 2024-12-07T00:51:31,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691467 2024-12-07T00:51:31,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691467) roll requested 2024-12-07T00:51:31,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691472 2024-12-07T00:51:31,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691472 2024-12-07T00:51:31,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691472) roll requested 2024-12-07T00:51:31,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691479 2024-12-07T00:51:31,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691479 2024-12-07T00:51:31,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691479) roll requested 2024-12-07T00:51:31,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691487 2024-12-07T00:51:31,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691487 2024-12-07T00:51:31,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691487) roll requested 2024-12-07T00:51:31,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691495 2024-12-07T00:51:31,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691495 2024-12-07T00:51:31,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691495) roll requested 2024-12-07T00:51:31,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691500 2024-12-07T00:51:31,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691500 2024-12-07T00:51:31,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691500) roll requested 2024-12-07T00:51:31,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691505 2024-12-07T00:51:31,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691505 2024-12-07T00:51:31,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691505) roll requested 2024-12-07T00:51:31,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691513 2024-12-07T00:51:31,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691513 2024-12-07T00:51:31,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691513) roll requested 2024-12-07T00:51:31,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691520 2024-12-07T00:51:31,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691520 2024-12-07T00:51:31,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691520) roll requested 2024-12-07T00:51:31,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691528 2024-12-07T00:51:31,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691528 2024-12-07T00:51:31,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691528) roll requested 2024-12-07T00:51:31,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691536 2024-12-07T00:51:31,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691536 2024-12-07T00:51:31,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691536) roll requested 2024-12-07T00:51:31,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691546 2024-12-07T00:51:31,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691546 2024-12-07T00:51:31,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691546) roll requested 2024-12-07T00:51:31,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691553 2024-12-07T00:51:31,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691553 2024-12-07T00:51:31,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691553) roll requested 2024-12-07T00:51:31,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691560 2024-12-07T00:51:31,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691560 2024-12-07T00:51:31,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691560) roll requested 2024-12-07T00:51:31,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691567 2024-12-07T00:51:31,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691567 2024-12-07T00:51:31,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691567) roll requested 2024-12-07T00:51:31,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691576 2024-12-07T00:51:31,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691576 2024-12-07T00:51:31,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691576) roll requested 2024-12-07T00:51:31,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691585 2024-12-07T00:51:31,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691585 2024-12-07T00:51:31,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691585) roll requested 2024-12-07T00:51:31,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691593 2024-12-07T00:51:31,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691593 2024-12-07T00:51:31,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691593) roll requested 2024-12-07T00:51:31,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691605 2024-12-07T00:51:31,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691605 2024-12-07T00:51:31,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691605) roll requested 2024-12-07T00:51:31,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691616 2024-12-07T00:51:31,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691616 2024-12-07T00:51:31,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691616) roll requested 2024-12-07T00:51:31,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691622 2024-12-07T00:51:31,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691622 2024-12-07T00:51:31,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691622) roll requested 2024-12-07T00:51:31,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691626 2024-12-07T00:51:31,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691626 2024-12-07T00:51:31,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691626) roll requested 2024-12-07T00:51:31,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691631 2024-12-07T00:51:31,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691631 2024-12-07T00:51:31,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691631) roll requested 2024-12-07T00:51:31,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691637 2024-12-07T00:51:31,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691637 2024-12-07T00:51:31,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691637) roll requested 2024-12-07T00:51:31,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691644 2024-12-07T00:51:31,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691644 2024-12-07T00:51:31,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691644) roll requested 2024-12-07T00:51:31,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691652 2024-12-07T00:51:31,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691652 2024-12-07T00:51:31,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691652) roll requested 2024-12-07T00:51:31,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691661 2024-12-07T00:51:31,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691661 2024-12-07T00:51:31,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691661) roll requested 2024-12-07T00:51:31,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691669 2024-12-07T00:51:31,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691669 2024-12-07T00:51:31,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691669) roll requested 2024-12-07T00:51:31,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691679 2024-12-07T00:51:31,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691679 2024-12-07T00:51:31,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691679) roll requested 2024-12-07T00:51:31,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691688 2024-12-07T00:51:31,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691688 2024-12-07T00:51:31,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691688) roll requested 2024-12-07T00:51:31,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691697 2024-12-07T00:51:31,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691697 2024-12-07T00:51:31,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691697) roll requested 2024-12-07T00:51:31,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691706 2024-12-07T00:51:31,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691706 2024-12-07T00:51:31,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691706) roll requested 2024-12-07T00:51:31,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691717 2024-12-07T00:51:31,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691717 2024-12-07T00:51:31,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691717) roll requested 2024-12-07T00:51:31,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691726 2024-12-07T00:51:31,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691726 2024-12-07T00:51:31,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691726) roll requested 2024-12-07T00:51:31,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691735 2024-12-07T00:51:31,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691735 2024-12-07T00:51:31,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691735) roll requested 2024-12-07T00:51:31,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691744 2024-12-07T00:51:31,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691744 2024-12-07T00:51:31,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691744) roll requested 2024-12-07T00:51:31,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691753 2024-12-07T00:51:31,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691753 2024-12-07T00:51:31,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691753) roll requested 2024-12-07T00:51:31,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691762 2024-12-07T00:51:31,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691762 2024-12-07T00:51:31,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691762) roll requested 2024-12-07T00:51:31,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691768 2024-12-07T00:51:31,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691768 2024-12-07T00:51:31,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691768) roll requested 2024-12-07T00:51:31,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691775 2024-12-07T00:51:31,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691775 2024-12-07T00:51:31,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691775) roll requested 2024-12-07T00:51:31,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691784 2024-12-07T00:51:31,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691784 2024-12-07T00:51:31,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691784) roll requested 2024-12-07T00:51:31,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691793 2024-12-07T00:51:31,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691793 2024-12-07T00:51:31,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691793) roll requested 2024-12-07T00:51:31,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691799 2024-12-07T00:51:31,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691799 2024-12-07T00:51:31,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691799) roll requested 2024-12-07T00:51:31,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691806 2024-12-07T00:51:31,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691806 2024-12-07T00:51:31,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691806) roll requested 2024-12-07T00:51:31,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691813 2024-12-07T00:51:31,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691813 2024-12-07T00:51:31,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691813) roll requested 2024-12-07T00:51:31,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691822 2024-12-07T00:51:31,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691822 2024-12-07T00:51:31,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691822) roll requested 2024-12-07T00:51:31,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691829 2024-12-07T00:51:31,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691829 2024-12-07T00:51:31,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691829) roll requested 2024-12-07T00:51:31,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691837 2024-12-07T00:51:31,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691837 2024-12-07T00:51:31,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691837) roll requested 2024-12-07T00:51:31,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691844 2024-12-07T00:51:31,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691844 2024-12-07T00:51:31,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691844) roll requested 2024-12-07T00:51:31,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691849 2024-12-07T00:51:31,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691849 2024-12-07T00:51:31,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691849) roll requested 2024-12-07T00:51:31,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691853 2024-12-07T00:51:31,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691853 2024-12-07T00:51:31,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691853) roll requested 2024-12-07T00:51:31,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691859 2024-12-07T00:51:31,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691859 2024-12-07T00:51:31,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691859) roll requested 2024-12-07T00:51:31,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691866 2024-12-07T00:51:31,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691866 2024-12-07T00:51:31,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691866) roll requested 2024-12-07T00:51:31,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691873 2024-12-07T00:51:31,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691873 2024-12-07T00:51:31,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691873) roll requested 2024-12-07T00:51:31,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691881 2024-12-07T00:51:31,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691881 2024-12-07T00:51:31,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691881) roll requested 2024-12-07T00:51:31,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691888 2024-12-07T00:51:31,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691888 2024-12-07T00:51:31,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691888) roll requested 2024-12-07T00:51:31,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691898 2024-12-07T00:51:31,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691898 2024-12-07T00:51:31,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691898) roll requested 2024-12-07T00:51:31,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691907 2024-12-07T00:51:31,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691907 2024-12-07T00:51:31,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691907) roll requested 2024-12-07T00:51:31,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691915 2024-12-07T00:51:31,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691915 2024-12-07T00:51:31,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691915) roll requested 2024-12-07T00:51:31,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691924 2024-12-07T00:51:31,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691924 2024-12-07T00:51:31,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691924) roll requested 2024-12-07T00:51:31,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691933 2024-12-07T00:51:31,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691933 2024-12-07T00:51:31,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691933) roll requested 2024-12-07T00:51:31,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691943 2024-12-07T00:51:31,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691943 2024-12-07T00:51:31,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691943) roll requested 2024-12-07T00:51:31,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691951 2024-12-07T00:51:31,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691951 2024-12-07T00:51:31,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691951) roll requested 2024-12-07T00:51:31,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691960 2024-12-07T00:51:31,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691960 2024-12-07T00:51:31,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691960) roll requested 2024-12-07T00:51:31,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691969 2024-12-07T00:51:31,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691969 2024-12-07T00:51:31,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691969) roll requested 2024-12-07T00:51:31,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691978 2024-12-07T00:51:31,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691978 2024-12-07T00:51:31,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691978) roll requested 2024-12-07T00:51:31,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691987 2024-12-07T00:51:31,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:31,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691987 2024-12-07T00:51:31,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:31,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:31,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691987) roll requested 2024-12-07T00:51:31,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532691996 2024-12-07T00:51:32,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691996 2024-12-07T00:51:32,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532691996) roll requested 2024-12-07T00:51:32,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692005 2024-12-07T00:51:32,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532691996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692005 2024-12-07T00:51:32,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692005) roll requested 2024-12-07T00:51:32,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692014 2024-12-07T00:51:32,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692014 2024-12-07T00:51:32,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692014) roll requested 2024-12-07T00:51:32,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692022 2024-12-07T00:51:32,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692022 2024-12-07T00:51:32,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692022) roll requested 2024-12-07T00:51:32,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692031 2024-12-07T00:51:32,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692031 2024-12-07T00:51:32,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692031) roll requested 2024-12-07T00:51:32,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692040 2024-12-07T00:51:32,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692040 2024-12-07T00:51:32,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692040) roll requested 2024-12-07T00:51:32,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692048 2024-12-07T00:51:32,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692048 2024-12-07T00:51:32,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692048) roll requested 2024-12-07T00:51:32,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692057 2024-12-07T00:51:32,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692057 2024-12-07T00:51:32,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692057) roll requested 2024-12-07T00:51:32,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692067 2024-12-07T00:51:32,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692067 2024-12-07T00:51:32,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692067) roll requested 2024-12-07T00:51:32,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692076 2024-12-07T00:51:32,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692076 2024-12-07T00:51:32,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692076) roll requested 2024-12-07T00:51:32,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692081 2024-12-07T00:51:32,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692081 2024-12-07T00:51:32,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692081) roll requested 2024-12-07T00:51:32,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692087 2024-12-07T00:51:32,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692087 2024-12-07T00:51:32,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692087) roll requested 2024-12-07T00:51:32,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692092 2024-12-07T00:51:32,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692092 2024-12-07T00:51:32,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692092) roll requested 2024-12-07T00:51:32,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692098 2024-12-07T00:51:32,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692098 2024-12-07T00:51:32,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692098) roll requested 2024-12-07T00:51:32,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692105 2024-12-07T00:51:32,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692105 2024-12-07T00:51:32,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692105) roll requested 2024-12-07T00:51:32,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692114 2024-12-07T00:51:32,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692114 2024-12-07T00:51:32,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692114) roll requested 2024-12-07T00:51:32,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692124 2024-12-07T00:51:32,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692124 2024-12-07T00:51:32,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692124) roll requested 2024-12-07T00:51:32,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692134 2024-12-07T00:51:32,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692134 2024-12-07T00:51:32,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692134) roll requested 2024-12-07T00:51:32,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692145 2024-12-07T00:51:32,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692145 2024-12-07T00:51:32,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692145) roll requested 2024-12-07T00:51:32,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692152 2024-12-07T00:51:32,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692152 2024-12-07T00:51:32,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692152) roll requested 2024-12-07T00:51:32,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692157 2024-12-07T00:51:32,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692157 2024-12-07T00:51:32,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692157) roll requested 2024-12-07T00:51:32,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692163 2024-12-07T00:51:32,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692163 2024-12-07T00:51:32,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692163) roll requested 2024-12-07T00:51:32,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692168 2024-12-07T00:51:32,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692168 2024-12-07T00:51:32,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692168) roll requested 2024-12-07T00:51:32,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692174 2024-12-07T00:51:32,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692174 2024-12-07T00:51:32,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692174) roll requested 2024-12-07T00:51:32,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692181 2024-12-07T00:51:32,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692181 2024-12-07T00:51:32,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692181) roll requested 2024-12-07T00:51:32,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692188 2024-12-07T00:51:32,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692188 2024-12-07T00:51:32,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692188) roll requested 2024-12-07T00:51:32,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692196 2024-12-07T00:51:32,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692196 2024-12-07T00:51:32,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692196) roll requested 2024-12-07T00:51:32,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692204 2024-12-07T00:51:32,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692204 2024-12-07T00:51:32,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692204) roll requested 2024-12-07T00:51:32,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692213 2024-12-07T00:51:32,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692213 2024-12-07T00:51:32,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692213) roll requested 2024-12-07T00:51:32,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692222 2024-12-07T00:51:32,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692222 2024-12-07T00:51:32,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692222) roll requested 2024-12-07T00:51:32,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692231 2024-12-07T00:51:32,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692231 2024-12-07T00:51:32,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692231) roll requested 2024-12-07T00:51:32,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692239 2024-12-07T00:51:32,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692239 2024-12-07T00:51:32,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692239) roll requested 2024-12-07T00:51:32,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692248 2024-12-07T00:51:32,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692248 2024-12-07T00:51:32,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692248) roll requested 2024-12-07T00:51:32,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692254 2024-12-07T00:51:32,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692254 2024-12-07T00:51:32,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692254) roll requested 2024-12-07T00:51:32,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692262 2024-12-07T00:51:32,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692262 2024-12-07T00:51:32,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692262) roll requested 2024-12-07T00:51:32,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692270 2024-12-07T00:51:32,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692270 2024-12-07T00:51:32,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692270) roll requested 2024-12-07T00:51:32,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692278 2024-12-07T00:51:32,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692278 2024-12-07T00:51:32,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692278) roll requested 2024-12-07T00:51:32,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692288 2024-12-07T00:51:32,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692288 2024-12-07T00:51:32,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692288) roll requested 2024-12-07T00:51:32,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692298 2024-12-07T00:51:32,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692298 2024-12-07T00:51:32,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692298) roll requested 2024-12-07T00:51:32,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692306 2024-12-07T00:51:32,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692306 2024-12-07T00:51:32,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692306) roll requested 2024-12-07T00:51:32,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692313 2024-12-07T00:51:32,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692313 2024-12-07T00:51:32,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692313) roll requested 2024-12-07T00:51:32,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692319 2024-12-07T00:51:32,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692319 2024-12-07T00:51:32,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692319) roll requested 2024-12-07T00:51:32,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692326 2024-12-07T00:51:32,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692326 2024-12-07T00:51:32,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692326) roll requested 2024-12-07T00:51:32,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692334 2024-12-07T00:51:32,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692334 2024-12-07T00:51:32,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692334) roll requested 2024-12-07T00:51:32,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692342 2024-12-07T00:51:32,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692342 2024-12-07T00:51:32,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692342) roll requested 2024-12-07T00:51:32,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692351 2024-12-07T00:51:32,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692351 2024-12-07T00:51:32,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692351) roll requested 2024-12-07T00:51:32,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692360 2024-12-07T00:51:32,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692360 2024-12-07T00:51:32,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692360) roll requested 2024-12-07T00:51:32,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692369 2024-12-07T00:51:32,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692369 2024-12-07T00:51:32,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692369) roll requested 2024-12-07T00:51:32,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692379 2024-12-07T00:51:32,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692379 2024-12-07T00:51:32,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692379) roll requested 2024-12-07T00:51:32,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692390 2024-12-07T00:51:32,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692390 2024-12-07T00:51:32,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692390) roll requested 2024-12-07T00:51:32,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692399 2024-12-07T00:51:32,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692399 2024-12-07T00:51:32,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692399) roll requested 2024-12-07T00:51:32,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692407 2024-12-07T00:51:32,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692407 2024-12-07T00:51:32,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692407) roll requested 2024-12-07T00:51:32,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692414 2024-12-07T00:51:32,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692414 2024-12-07T00:51:32,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692414) roll requested 2024-12-07T00:51:32,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692420 2024-12-07T00:51:32,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692420 2024-12-07T00:51:32,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692420) roll requested 2024-12-07T00:51:32,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692430 2024-12-07T00:51:32,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692430 2024-12-07T00:51:32,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692430) roll requested 2024-12-07T00:51:32,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692436 2024-12-07T00:51:32,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692436 2024-12-07T00:51:32,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692436) roll requested 2024-12-07T00:51:32,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692441 2024-12-07T00:51:32,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692441 2024-12-07T00:51:32,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692441) roll requested 2024-12-07T00:51:32,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692447 2024-12-07T00:51:32,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692447 2024-12-07T00:51:32,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692447) roll requested 2024-12-07T00:51:32,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692454 2024-12-07T00:51:32,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692454 2024-12-07T00:51:32,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692454) roll requested 2024-12-07T00:51:32,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692461 2024-12-07T00:51:32,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692461 2024-12-07T00:51:32,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692461) roll requested 2024-12-07T00:51:32,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692469 2024-12-07T00:51:32,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692469 2024-12-07T00:51:32,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692469) roll requested 2024-12-07T00:51:32,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692478 2024-12-07T00:51:32,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692478 2024-12-07T00:51:32,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692478) roll requested 2024-12-07T00:51:32,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692487 2024-12-07T00:51:32,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692487 2024-12-07T00:51:32,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692487) roll requested 2024-12-07T00:51:32,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692496 2024-12-07T00:51:32,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692496 2024-12-07T00:51:32,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692496) roll requested 2024-12-07T00:51:32,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692503 2024-12-07T00:51:32,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692503 2024-12-07T00:51:32,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692503) roll requested 2024-12-07T00:51:32,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692510 2024-12-07T00:51:32,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692510 2024-12-07T00:51:32,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692510) roll requested 2024-12-07T00:51:32,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692519 2024-12-07T00:51:32,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692519 2024-12-07T00:51:32,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692519) roll requested 2024-12-07T00:51:32,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692526 2024-12-07T00:51:32,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692526 2024-12-07T00:51:32,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692526) roll requested 2024-12-07T00:51:32,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692533 2024-12-07T00:51:32,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692533 2024-12-07T00:51:32,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692533) roll requested 2024-12-07T00:51:32,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692543 2024-12-07T00:51:32,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692543 2024-12-07T00:51:32,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692543) roll requested 2024-12-07T00:51:32,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692553 2024-12-07T00:51:32,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692553 2024-12-07T00:51:32,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692553) roll requested 2024-12-07T00:51:32,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692560 2024-12-07T00:51:32,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692560 2024-12-07T00:51:32,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692560) roll requested 2024-12-07T00:51:32,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692565 2024-12-07T00:51:32,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692565 2024-12-07T00:51:32,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692565) roll requested 2024-12-07T00:51:32,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692571 2024-12-07T00:51:32,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692571 2024-12-07T00:51:32,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692571) roll requested 2024-12-07T00:51:32,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692577 2024-12-07T00:51:32,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692577 2024-12-07T00:51:32,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692577) roll requested 2024-12-07T00:51:32,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692583 2024-12-07T00:51:32,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692583 2024-12-07T00:51:32,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692583) roll requested 2024-12-07T00:51:32,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692591 2024-12-07T00:51:32,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692591 2024-12-07T00:51:32,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692591) roll requested 2024-12-07T00:51:32,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692599 2024-12-07T00:51:32,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692599 2024-12-07T00:51:32,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692599) roll requested 2024-12-07T00:51:32,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692609 2024-12-07T00:51:32,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692609 2024-12-07T00:51:32,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692609) roll requested 2024-12-07T00:51:32,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692618 2024-12-07T00:51:32,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692618 2024-12-07T00:51:32,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692618) roll requested 2024-12-07T00:51:32,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692625 2024-12-07T00:51:32,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692625 2024-12-07T00:51:32,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692625) roll requested 2024-12-07T00:51:32,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692633 2024-12-07T00:51:32,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692633 2024-12-07T00:51:32,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692633) roll requested 2024-12-07T00:51:32,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692640 2024-12-07T00:51:32,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692640 2024-12-07T00:51:32,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692640) roll requested 2024-12-07T00:51:32,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692646 2024-12-07T00:51:32,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692646 2024-12-07T00:51:32,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692646) roll requested 2024-12-07T00:51:32,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692655 2024-12-07T00:51:32,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692655 2024-12-07T00:51:32,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692655) roll requested 2024-12-07T00:51:32,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692662 2024-12-07T00:51:32,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692662 2024-12-07T00:51:32,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692662) roll requested 2024-12-07T00:51:32,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692668 2024-12-07T00:51:32,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692668 2024-12-07T00:51:32,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692668) roll requested 2024-12-07T00:51:32,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692676 2024-12-07T00:51:32,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692676 2024-12-07T00:51:32,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692676) roll requested 2024-12-07T00:51:32,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692684 2024-12-07T00:51:32,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692684 2024-12-07T00:51:32,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692684) roll requested 2024-12-07T00:51:32,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692692 2024-12-07T00:51:32,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692692 2024-12-07T00:51:32,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692692) roll requested 2024-12-07T00:51:32,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692697 2024-12-07T00:51:32,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692697 2024-12-07T00:51:32,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692697) roll requested 2024-12-07T00:51:32,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692702 2024-12-07T00:51:32,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692702 2024-12-07T00:51:32,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692702) roll requested 2024-12-07T00:51:32,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692710 2024-12-07T00:51:32,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692710 2024-12-07T00:51:32,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692710) roll requested 2024-12-07T00:51:32,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692718 2024-12-07T00:51:32,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692718 2024-12-07T00:51:32,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692718) roll requested 2024-12-07T00:51:32,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692727 2024-12-07T00:51:32,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692727 2024-12-07T00:51:32,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692727) roll requested 2024-12-07T00:51:32,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692737 2024-12-07T00:51:32,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692737 2024-12-07T00:51:32,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692737) roll requested 2024-12-07T00:51:32,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692747 2024-12-07T00:51:32,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692747 2024-12-07T00:51:32,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692747) roll requested 2024-12-07T00:51:32,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692754 2024-12-07T00:51:32,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692754 2024-12-07T00:51:32,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692754) roll requested 2024-12-07T00:51:32,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692760 2024-12-07T00:51:32,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692760 2024-12-07T00:51:32,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692760) roll requested 2024-12-07T00:51:32,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692768 2024-12-07T00:51:32,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692768 2024-12-07T00:51:32,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692768) roll requested 2024-12-07T00:51:32,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692776 2024-12-07T00:51:32,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692776 2024-12-07T00:51:32,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692776) roll requested 2024-12-07T00:51:32,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692785 2024-12-07T00:51:32,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692785 2024-12-07T00:51:32,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692785) roll requested 2024-12-07T00:51:32,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692794 2024-12-07T00:51:32,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692794 2024-12-07T00:51:32,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692794) roll requested 2024-12-07T00:51:32,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692803 2024-12-07T00:51:32,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692803 2024-12-07T00:51:32,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692803) roll requested 2024-12-07T00:51:32,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692811 2024-12-07T00:51:32,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692811 2024-12-07T00:51:32,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692811) roll requested 2024-12-07T00:51:32,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692819 2024-12-07T00:51:32,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692819 2024-12-07T00:51:32,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692819) roll requested 2024-12-07T00:51:32,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692825 2024-12-07T00:51:32,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692825 2024-12-07T00:51:32,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692825) roll requested 2024-12-07T00:51:32,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692833 2024-12-07T00:51:32,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692833 2024-12-07T00:51:32,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692833) roll requested 2024-12-07T00:51:32,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692839 2024-12-07T00:51:32,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692839 2024-12-07T00:51:32,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692839) roll requested 2024-12-07T00:51:32,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692845 2024-12-07T00:51:32,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692845 2024-12-07T00:51:32,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692845) roll requested 2024-12-07T00:51:32,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692850 2024-12-07T00:51:32,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692850 2024-12-07T00:51:32,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692850) roll requested 2024-12-07T00:51:32,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692856 2024-12-07T00:51:32,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692856 2024-12-07T00:51:32,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692856) roll requested 2024-12-07T00:51:32,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692864 2024-12-07T00:51:32,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692864 2024-12-07T00:51:32,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692864) roll requested 2024-12-07T00:51:32,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692872 2024-12-07T00:51:32,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692872 2024-12-07T00:51:32,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692872) roll requested 2024-12-07T00:51:32,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692877 2024-12-07T00:51:32,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692877 2024-12-07T00:51:32,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692877) roll requested 2024-12-07T00:51:32,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692883 2024-12-07T00:51:32,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692883 2024-12-07T00:51:32,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692883) roll requested 2024-12-07T00:51:32,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692888 2024-12-07T00:51:32,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692888 2024-12-07T00:51:32,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692888) roll requested 2024-12-07T00:51:32,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692893 2024-12-07T00:51:32,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692893 2024-12-07T00:51:32,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692893) roll requested 2024-12-07T00:51:32,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692899 2024-12-07T00:51:32,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692899 2024-12-07T00:51:32,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692899) roll requested 2024-12-07T00:51:32,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692904 2024-12-07T00:51:32,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692904 2024-12-07T00:51:32,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692904) roll requested 2024-12-07T00:51:32,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692909 2024-12-07T00:51:32,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692909 2024-12-07T00:51:32,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692909) roll requested 2024-12-07T00:51:32,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692916 2024-12-07T00:51:32,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692916 2024-12-07T00:51:32,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692916) roll requested 2024-12-07T00:51:32,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692929 2024-12-07T00:51:32,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692929 2024-12-07T00:51:32,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692929) roll requested 2024-12-07T00:51:32,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692934 2024-12-07T00:51:32,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692934 2024-12-07T00:51:32,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692934) roll requested 2024-12-07T00:51:32,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692940 2024-12-07T00:51:32,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692940 2024-12-07T00:51:32,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692940) roll requested 2024-12-07T00:51:32,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692945 2024-12-07T00:51:32,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692945 2024-12-07T00:51:32,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692945) roll requested 2024-12-07T00:51:32,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692953 2024-12-07T00:51:32,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692953 2024-12-07T00:51:32,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692953) roll requested 2024-12-07T00:51:32,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692959 2024-12-07T00:51:32,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692959 2024-12-07T00:51:32,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692959) roll requested 2024-12-07T00:51:32,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692964 2024-12-07T00:51:32,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692964 2024-12-07T00:51:32,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692964) roll requested 2024-12-07T00:51:32,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692969 2024-12-07T00:51:32,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692969 2024-12-07T00:51:32,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692969) roll requested 2024-12-07T00:51:32,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692982 2024-12-07T00:51:32,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692982 2024-12-07T00:51:32,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692982) roll requested 2024-12-07T00:51:32,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692987 2024-12-07T00:51:32,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692987 2024-12-07T00:51:32,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692987) roll requested 2024-12-07T00:51:32,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692993 2024-12-07T00:51:32,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:32,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692993 2024-12-07T00:51:32,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:32,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:32,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692993) roll requested 2024-12-07T00:51:32,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532692998 2024-12-07T00:51:33,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692998 2024-12-07T00:51:33,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532692998) roll requested 2024-12-07T00:51:33,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693003 2024-12-07T00:51:33,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532692998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693003 2024-12-07T00:51:33,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693003) roll requested 2024-12-07T00:51:33,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693012 2024-12-07T00:51:33,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693012 2024-12-07T00:51:33,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693012) roll requested 2024-12-07T00:51:33,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693017 2024-12-07T00:51:33,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693017 2024-12-07T00:51:33,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693017) roll requested 2024-12-07T00:51:33,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693023 2024-12-07T00:51:33,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693023 2024-12-07T00:51:33,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693023) roll requested 2024-12-07T00:51:33,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693028 2024-12-07T00:51:33,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693028 2024-12-07T00:51:33,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693028) roll requested 2024-12-07T00:51:33,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693033 2024-12-07T00:51:33,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693033 2024-12-07T00:51:33,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693033) roll requested 2024-12-07T00:51:33,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693039 2024-12-07T00:51:33,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693039 2024-12-07T00:51:33,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693039) roll requested 2024-12-07T00:51:33,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693046 2024-12-07T00:51:33,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693046 2024-12-07T00:51:33,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693046) roll requested 2024-12-07T00:51:33,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693054 2024-12-07T00:51:33,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693054 2024-12-07T00:51:33,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693054) roll requested 2024-12-07T00:51:33,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693062 2024-12-07T00:51:33,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693062 2024-12-07T00:51:33,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693062) roll requested 2024-12-07T00:51:33,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693072 2024-12-07T00:51:33,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693072 2024-12-07T00:51:33,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693072) roll requested 2024-12-07T00:51:33,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693077 2024-12-07T00:51:33,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693077 2024-12-07T00:51:33,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693077) roll requested 2024-12-07T00:51:33,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693083 2024-12-07T00:51:33,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693083 2024-12-07T00:51:33,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693083) roll requested 2024-12-07T00:51:33,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693090 2024-12-07T00:51:33,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693090 2024-12-07T00:51:33,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693090) roll requested 2024-12-07T00:51:33,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693098 2024-12-07T00:51:33,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693098 2024-12-07T00:51:33,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693098) roll requested 2024-12-07T00:51:33,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693106 2024-12-07T00:51:33,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693106 2024-12-07T00:51:33,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693106) roll requested 2024-12-07T00:51:33,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693113 2024-12-07T00:51:33,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693113 2024-12-07T00:51:33,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693113) roll requested 2024-12-07T00:51:33,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693120 2024-12-07T00:51:33,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693120 2024-12-07T00:51:33,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693120) roll requested 2024-12-07T00:51:33,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693129 2024-12-07T00:51:33,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693129 2024-12-07T00:51:33,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693129) roll requested 2024-12-07T00:51:33,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693137 2024-12-07T00:51:33,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693137 2024-12-07T00:51:33,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693137) roll requested 2024-12-07T00:51:33,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693146 2024-12-07T00:51:33,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693146 2024-12-07T00:51:33,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693146) roll requested 2024-12-07T00:51:33,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693155 2024-12-07T00:51:33,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693155 2024-12-07T00:51:33,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693155) roll requested 2024-12-07T00:51:33,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693167 2024-12-07T00:51:33,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693167 2024-12-07T00:51:33,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693167) roll requested 2024-12-07T00:51:33,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693172 2024-12-07T00:51:33,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693172 2024-12-07T00:51:33,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693172) roll requested 2024-12-07T00:51:33,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693177 2024-12-07T00:51:33,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693177 2024-12-07T00:51:33,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693177) roll requested 2024-12-07T00:51:33,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693183 2024-12-07T00:51:33,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693183 2024-12-07T00:51:33,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693183) roll requested 2024-12-07T00:51:33,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693190 2024-12-07T00:51:33,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693190 2024-12-07T00:51:33,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693190) roll requested 2024-12-07T00:51:33,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693198 2024-12-07T00:51:33,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693198 2024-12-07T00:51:33,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693198) roll requested 2024-12-07T00:51:33,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693205 2024-12-07T00:51:33,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693205 2024-12-07T00:51:33,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693205) roll requested 2024-12-07T00:51:33,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693214 2024-12-07T00:51:33,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693214 2024-12-07T00:51:33,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693214) roll requested 2024-12-07T00:51:33,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693223 2024-12-07T00:51:33,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693223 2024-12-07T00:51:33,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693223) roll requested 2024-12-07T00:51:33,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693232 2024-12-07T00:51:33,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693232 2024-12-07T00:51:33,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693232) roll requested 2024-12-07T00:51:33,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693242 2024-12-07T00:51:33,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693242 2024-12-07T00:51:33,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693242) roll requested 2024-12-07T00:51:33,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693252 2024-12-07T00:51:33,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693252 2024-12-07T00:51:33,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693252) roll requested 2024-12-07T00:51:33,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693260 2024-12-07T00:51:33,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693260 2024-12-07T00:51:33,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693260) roll requested 2024-12-07T00:51:33,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693267 2024-12-07T00:51:33,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693267 2024-12-07T00:51:33,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693267) roll requested 2024-12-07T00:51:33,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693276 2024-12-07T00:51:33,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693276 2024-12-07T00:51:33,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693276) roll requested 2024-12-07T00:51:33,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693284 2024-12-07T00:51:33,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693284 2024-12-07T00:51:33,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693284) roll requested 2024-12-07T00:51:33,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693293 2024-12-07T00:51:33,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693293 2024-12-07T00:51:33,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693293) roll requested 2024-12-07T00:51:33,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693300 2024-12-07T00:51:33,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693300 2024-12-07T00:51:33,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693300) roll requested 2024-12-07T00:51:33,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693305 2024-12-07T00:51:33,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693305 2024-12-07T00:51:33,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693305) roll requested 2024-12-07T00:51:33,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693314 2024-12-07T00:51:33,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693314 2024-12-07T00:51:33,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693314) roll requested 2024-12-07T00:51:33,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693375 2024-12-07T00:51:33,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693375 2024-12-07T00:51:33,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693375) roll requested 2024-12-07T00:51:33,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693387 2024-12-07T00:51:33,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693387 2024-12-07T00:51:33,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693387) roll requested 2024-12-07T00:51:33,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693394 2024-12-07T00:51:33,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693394 2024-12-07T00:51:33,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693394) roll requested 2024-12-07T00:51:33,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693400 2024-12-07T00:51:33,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693400 2024-12-07T00:51:33,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693400) roll requested 2024-12-07T00:51:33,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693407 2024-12-07T00:51:33,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693407 2024-12-07T00:51:33,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693407) roll requested 2024-12-07T00:51:33,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693416 2024-12-07T00:51:33,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693416 2024-12-07T00:51:33,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693416) roll requested 2024-12-07T00:51:33,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693425 2024-12-07T00:51:33,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693425 2024-12-07T00:51:33,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693425) roll requested 2024-12-07T00:51:33,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693432 2024-12-07T00:51:33,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693432 2024-12-07T00:51:33,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693432) roll requested 2024-12-07T00:51:33,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693441 2024-12-07T00:51:33,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693441 2024-12-07T00:51:33,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693441) roll requested 2024-12-07T00:51:33,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693450 2024-12-07T00:51:33,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693450 2024-12-07T00:51:33,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693450) roll requested 2024-12-07T00:51:33,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693460 2024-12-07T00:51:33,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693460 2024-12-07T00:51:33,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693460) roll requested 2024-12-07T00:51:33,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693468 2024-12-07T00:51:33,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693468 2024-12-07T00:51:33,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693468) roll requested 2024-12-07T00:51:33,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693473 2024-12-07T00:51:33,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693473 2024-12-07T00:51:33,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693473) roll requested 2024-12-07T00:51:33,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693479 2024-12-07T00:51:33,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693479 2024-12-07T00:51:33,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693479) roll requested 2024-12-07T00:51:33,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693486 2024-12-07T00:51:33,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693486 2024-12-07T00:51:33,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693486) roll requested 2024-12-07T00:51:33,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693493 2024-12-07T00:51:33,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693493 2024-12-07T00:51:33,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693493) roll requested 2024-12-07T00:51:33,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693501 2024-12-07T00:51:33,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693501 2024-12-07T00:51:33,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693501) roll requested 2024-12-07T00:51:33,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693509 2024-12-07T00:51:33,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693509 2024-12-07T00:51:33,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693509) roll requested 2024-12-07T00:51:33,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693518 2024-12-07T00:51:33,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693518 2024-12-07T00:51:33,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693518) roll requested 2024-12-07T00:51:33,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693528 2024-12-07T00:51:33,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693528 2024-12-07T00:51:33,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693528) roll requested 2024-12-07T00:51:33,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693533 2024-12-07T00:51:33,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693533 2024-12-07T00:51:33,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693533) roll requested 2024-12-07T00:51:33,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693539 2024-12-07T00:51:33,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693539 2024-12-07T00:51:33,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693539) roll requested 2024-12-07T00:51:33,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693547 2024-12-07T00:51:33,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693547 2024-12-07T00:51:33,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693547) roll requested 2024-12-07T00:51:33,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693555 2024-12-07T00:51:33,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693555 2024-12-07T00:51:33,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693555) roll requested 2024-12-07T00:51:33,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693568 2024-12-07T00:51:33,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693568 2024-12-07T00:51:33,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693568) roll requested 2024-12-07T00:51:33,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693575 2024-12-07T00:51:33,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693575 2024-12-07T00:51:33,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693575) roll requested 2024-12-07T00:51:33,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693579 2024-12-07T00:51:33,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693579 2024-12-07T00:51:33,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693579) roll requested 2024-12-07T00:51:33,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693584 2024-12-07T00:51:33,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693584 2024-12-07T00:51:33,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693584) roll requested 2024-12-07T00:51:33,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693591 2024-12-07T00:51:33,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693591 2024-12-07T00:51:33,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693591) roll requested 2024-12-07T00:51:33,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693599 2024-12-07T00:51:33,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693599 2024-12-07T00:51:33,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693599) roll requested 2024-12-07T00:51:33,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693606 2024-12-07T00:51:33,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693606 2024-12-07T00:51:33,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693606) roll requested 2024-12-07T00:51:33,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693614 2024-12-07T00:51:33,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693614 2024-12-07T00:51:33,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693614) roll requested 2024-12-07T00:51:33,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693622 2024-12-07T00:51:33,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693622 2024-12-07T00:51:33,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693622) roll requested 2024-12-07T00:51:33,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693632 2024-12-07T00:51:33,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693632 2024-12-07T00:51:33,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693632) roll requested 2024-12-07T00:51:33,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693641 2024-12-07T00:51:33,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693641 2024-12-07T00:51:33,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693641) roll requested 2024-12-07T00:51:33,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693648 2024-12-07T00:51:33,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693648 2024-12-07T00:51:33,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693648) roll requested 2024-12-07T00:51:33,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693655 2024-12-07T00:51:33,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693655 2024-12-07T00:51:33,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693655) roll requested 2024-12-07T00:51:33,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693662 2024-12-07T00:51:33,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693662 2024-12-07T00:51:33,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693662) roll requested 2024-12-07T00:51:33,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693670 2024-12-07T00:51:33,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693670 2024-12-07T00:51:33,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693670) roll requested 2024-12-07T00:51:33,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693677 2024-12-07T00:51:33,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693677 2024-12-07T00:51:33,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693677) roll requested 2024-12-07T00:51:33,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693685 2024-12-07T00:51:33,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693685 2024-12-07T00:51:33,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693685) roll requested 2024-12-07T00:51:33,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693694 2024-12-07T00:51:33,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693694 2024-12-07T00:51:33,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693694) roll requested 2024-12-07T00:51:33,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693700 2024-12-07T00:51:33,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693700 2024-12-07T00:51:33,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693700) roll requested 2024-12-07T00:51:33,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693707 2024-12-07T00:51:33,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693707 2024-12-07T00:51:33,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693707) roll requested 2024-12-07T00:51:33,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693715 2024-12-07T00:51:33,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693715 2024-12-07T00:51:33,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693715) roll requested 2024-12-07T00:51:33,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693723 2024-12-07T00:51:33,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693723 2024-12-07T00:51:33,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693723) roll requested 2024-12-07T00:51:33,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693731 2024-12-07T00:51:33,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693731 2024-12-07T00:51:33,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693731) roll requested 2024-12-07T00:51:33,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693739 2024-12-07T00:51:33,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693739 2024-12-07T00:51:33,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693739) roll requested 2024-12-07T00:51:33,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693747 2024-12-07T00:51:33,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693747 2024-12-07T00:51:33,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693747) roll requested 2024-12-07T00:51:33,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693756 2024-12-07T00:51:33,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693756 2024-12-07T00:51:33,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693756) roll requested 2024-12-07T00:51:33,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693766 2024-12-07T00:51:33,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693766 2024-12-07T00:51:33,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693766) roll requested 2024-12-07T00:51:33,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693774 2024-12-07T00:51:33,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693774 2024-12-07T00:51:33,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693774) roll requested 2024-12-07T00:51:33,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693782 2024-12-07T00:51:33,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693782 2024-12-07T00:51:33,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693782) roll requested 2024-12-07T00:51:33,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693789 2024-12-07T00:51:33,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693789 2024-12-07T00:51:33,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693789) roll requested 2024-12-07T00:51:33,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693796 2024-12-07T00:51:33,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693796 2024-12-07T00:51:33,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693796) roll requested 2024-12-07T00:51:33,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693806 2024-12-07T00:51:33,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693806 2024-12-07T00:51:33,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693806) roll requested 2024-12-07T00:51:33,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693813 2024-12-07T00:51:33,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693813 2024-12-07T00:51:33,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693813) roll requested 2024-12-07T00:51:33,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693818 2024-12-07T00:51:33,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693818 2024-12-07T00:51:33,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693818) roll requested 2024-12-07T00:51:33,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693825 2024-12-07T00:51:33,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693825 2024-12-07T00:51:33,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693825) roll requested 2024-12-07T00:51:33,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693833 2024-12-07T00:51:33,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693833 2024-12-07T00:51:33,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693833) roll requested 2024-12-07T00:51:33,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693839 2024-12-07T00:51:33,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693839 2024-12-07T00:51:33,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693839) roll requested 2024-12-07T00:51:33,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693846 2024-12-07T00:51:33,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693846 2024-12-07T00:51:33,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693846) roll requested 2024-12-07T00:51:33,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693853 2024-12-07T00:51:33,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693853 2024-12-07T00:51:33,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693853) roll requested 2024-12-07T00:51:33,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693861 2024-12-07T00:51:33,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693861 2024-12-07T00:51:33,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693861) roll requested 2024-12-07T00:51:33,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693869 2024-12-07T00:51:33,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693869 2024-12-07T00:51:33,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693869) roll requested 2024-12-07T00:51:33,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693878 2024-12-07T00:51:33,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693878 2024-12-07T00:51:33,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693878) roll requested 2024-12-07T00:51:33,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693885 2024-12-07T00:51:33,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693885 2024-12-07T00:51:33,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693885) roll requested 2024-12-07T00:51:33,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693893 2024-12-07T00:51:33,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693893 2024-12-07T00:51:33,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693893) roll requested 2024-12-07T00:51:33,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693900 2024-12-07T00:51:33,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693900 2024-12-07T00:51:33,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693900) roll requested 2024-12-07T00:51:33,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693908 2024-12-07T00:51:33,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693908 2024-12-07T00:51:33,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693908) roll requested 2024-12-07T00:51:33,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693916 2024-12-07T00:51:33,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693916 2024-12-07T00:51:33,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693916) roll requested 2024-12-07T00:51:33,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693925 2024-12-07T00:51:33,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693925 2024-12-07T00:51:33,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693925) roll requested 2024-12-07T00:51:33,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693932 2024-12-07T00:51:33,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693932 2024-12-07T00:51:33,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693932) roll requested 2024-12-07T00:51:33,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693938 2024-12-07T00:51:33,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693938 2024-12-07T00:51:33,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693938) roll requested 2024-12-07T00:51:33,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693945 2024-12-07T00:51:33,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693945 2024-12-07T00:51:33,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693945) roll requested 2024-12-07T00:51:33,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693953 2024-12-07T00:51:33,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693953 2024-12-07T00:51:33,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693953) roll requested 2024-12-07T00:51:33,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693960 2024-12-07T00:51:33,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693960 2024-12-07T00:51:33,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693960) roll requested 2024-12-07T00:51:33,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693965 2024-12-07T00:51:33,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693965 2024-12-07T00:51:33,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693965) roll requested 2024-12-07T00:51:33,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693970 2024-12-07T00:51:33,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693970 2024-12-07T00:51:33,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693970) roll requested 2024-12-07T00:51:33,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693978 2024-12-07T00:51:33,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693978 2024-12-07T00:51:33,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693978) roll requested 2024-12-07T00:51:33,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693986 2024-12-07T00:51:33,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:33,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693986 2024-12-07T00:51:33,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:33,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:33,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693986) roll requested 2024-12-07T00:51:33,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532693993 2024-12-07T00:51:34,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693993 2024-12-07T00:51:34,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532693993) roll requested 2024-12-07T00:51:34,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694002 2024-12-07T00:51:34,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532693993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694002 2024-12-07T00:51:34,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694002) roll requested 2024-12-07T00:51:34,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694012 2024-12-07T00:51:34,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694012 2024-12-07T00:51:34,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694012) roll requested 2024-12-07T00:51:34,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694019 2024-12-07T00:51:34,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694019 2024-12-07T00:51:34,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694019) roll requested 2024-12-07T00:51:34,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694026 2024-12-07T00:51:34,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694026 2024-12-07T00:51:34,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694026) roll requested 2024-12-07T00:51:34,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694034 2024-12-07T00:51:34,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694034 2024-12-07T00:51:34,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694034) roll requested 2024-12-07T00:51:34,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694043 2024-12-07T00:51:34,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694043 2024-12-07T00:51:34,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694043) roll requested 2024-12-07T00:51:34,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694051 2024-12-07T00:51:34,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694051 2024-12-07T00:51:34,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694051) roll requested 2024-12-07T00:51:34,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694059 2024-12-07T00:51:34,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694059 2024-12-07T00:51:34,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694059) roll requested 2024-12-07T00:51:34,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694069 2024-12-07T00:51:34,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694069 2024-12-07T00:51:34,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694069) roll requested 2024-12-07T00:51:34,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694077 2024-12-07T00:51:34,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694077 2024-12-07T00:51:34,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694077) roll requested 2024-12-07T00:51:34,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694085 2024-12-07T00:51:34,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694085 2024-12-07T00:51:34,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694085) roll requested 2024-12-07T00:51:34,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694093 2024-12-07T00:51:34,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694093 2024-12-07T00:51:34,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694093) roll requested 2024-12-07T00:51:34,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694103 2024-12-07T00:51:34,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694103 2024-12-07T00:51:34,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694103) roll requested 2024-12-07T00:51:34,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694111 2024-12-07T00:51:34,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694111 2024-12-07T00:51:34,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694111) roll requested 2024-12-07T00:51:34,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694119 2024-12-07T00:51:34,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694119 2024-12-07T00:51:34,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694119) roll requested 2024-12-07T00:51:34,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694126 2024-12-07T00:51:34,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694126 2024-12-07T00:51:34,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694126) roll requested 2024-12-07T00:51:34,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694134 2024-12-07T00:51:34,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694134 2024-12-07T00:51:34,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694134) roll requested 2024-12-07T00:51:34,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694142 2024-12-07T00:51:34,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694142 2024-12-07T00:51:34,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694142) roll requested 2024-12-07T00:51:34,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694149 2024-12-07T00:51:34,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694149 2024-12-07T00:51:34,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694149) roll requested 2024-12-07T00:51:34,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694158 2024-12-07T00:51:34,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694158 2024-12-07T00:51:34,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694158) roll requested 2024-12-07T00:51:34,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694166 2024-12-07T00:51:34,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694166 2024-12-07T00:51:34,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694166) roll requested 2024-12-07T00:51:34,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694173 2024-12-07T00:51:34,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694173 2024-12-07T00:51:34,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694173) roll requested 2024-12-07T00:51:34,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694180 2024-12-07T00:51:34,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694180 2024-12-07T00:51:34,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694180) roll requested 2024-12-07T00:51:34,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694189 2024-12-07T00:51:34,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694189 2024-12-07T00:51:34,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694189) roll requested 2024-12-07T00:51:34,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694197 2024-12-07T00:51:34,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694197 2024-12-07T00:51:34,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694197) roll requested 2024-12-07T00:51:34,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694206 2024-12-07T00:51:34,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694206 2024-12-07T00:51:34,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694206) roll requested 2024-12-07T00:51:34,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694215 2024-12-07T00:51:34,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694215 2024-12-07T00:51:34,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694215) roll requested 2024-12-07T00:51:34,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694224 2024-12-07T00:51:34,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694224 2024-12-07T00:51:34,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694224) roll requested 2024-12-07T00:51:34,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694232 2024-12-07T00:51:34,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694232 2024-12-07T00:51:34,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694232) roll requested 2024-12-07T00:51:34,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694238 2024-12-07T00:51:34,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694238 2024-12-07T00:51:34,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694238) roll requested 2024-12-07T00:51:34,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694245 2024-12-07T00:51:34,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694245 2024-12-07T00:51:34,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694245) roll requested 2024-12-07T00:51:34,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694254 2024-12-07T00:51:34,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694254 2024-12-07T00:51:34,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694254) roll requested 2024-12-07T00:51:34,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694263 2024-12-07T00:51:34,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694263 2024-12-07T00:51:34,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694263) roll requested 2024-12-07T00:51:34,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694273 2024-12-07T00:51:34,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694273 2024-12-07T00:51:34,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694273) roll requested 2024-12-07T00:51:34,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694282 2024-12-07T00:51:34,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694282 2024-12-07T00:51:34,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694282) roll requested 2024-12-07T00:51:34,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694289 2024-12-07T00:51:34,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694289 2024-12-07T00:51:34,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694289) roll requested 2024-12-07T00:51:34,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694298 2024-12-07T00:51:34,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694298 2024-12-07T00:51:34,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694298) roll requested 2024-12-07T00:51:34,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694305 2024-12-07T00:51:34,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694305 2024-12-07T00:51:34,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694305) roll requested 2024-12-07T00:51:34,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694311 2024-12-07T00:51:34,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694311 2024-12-07T00:51:34,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694311) roll requested 2024-12-07T00:51:34,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694319 2024-12-07T00:51:34,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694319 2024-12-07T00:51:34,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694319) roll requested 2024-12-07T00:51:34,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694327 2024-12-07T00:51:34,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694327 2024-12-07T00:51:34,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694327) roll requested 2024-12-07T00:51:34,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694336 2024-12-07T00:51:34,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694336 2024-12-07T00:51:34,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694336) roll requested 2024-12-07T00:51:34,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694345 2024-12-07T00:51:34,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694345 2024-12-07T00:51:34,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694345) roll requested 2024-12-07T00:51:34,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694355 2024-12-07T00:51:34,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694355 2024-12-07T00:51:34,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694355) roll requested 2024-12-07T00:51:34,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694363 2024-12-07T00:51:34,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694363 2024-12-07T00:51:34,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694363) roll requested 2024-12-07T00:51:34,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694371 2024-12-07T00:51:34,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694371 2024-12-07T00:51:34,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694371) roll requested 2024-12-07T00:51:34,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694379 2024-12-07T00:51:34,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694379 2024-12-07T00:51:34,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694379) roll requested 2024-12-07T00:51:34,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694387 2024-12-07T00:51:34,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694387 2024-12-07T00:51:34,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694387) roll requested 2024-12-07T00:51:34,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694394 2024-12-07T00:51:34,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694394 2024-12-07T00:51:34,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694394) roll requested 2024-12-07T00:51:34,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694400 2024-12-07T00:51:34,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694400 2024-12-07T00:51:34,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694400) roll requested 2024-12-07T00:51:34,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694407 2024-12-07T00:51:34,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694407 2024-12-07T00:51:34,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694407) roll requested 2024-12-07T00:51:34,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694414 2024-12-07T00:51:34,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694414 2024-12-07T00:51:34,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694414) roll requested 2024-12-07T00:51:34,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694422 2024-12-07T00:51:34,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694422 2024-12-07T00:51:34,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694422) roll requested 2024-12-07T00:51:34,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694431 2024-12-07T00:51:34,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694431 2024-12-07T00:51:34,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694431) roll requested 2024-12-07T00:51:34,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694441 2024-12-07T00:51:34,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694441 2024-12-07T00:51:34,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694441) roll requested 2024-12-07T00:51:34,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694450 2024-12-07T00:51:34,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694450 2024-12-07T00:51:34,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694450) roll requested 2024-12-07T00:51:34,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694456 2024-12-07T00:51:34,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694456 2024-12-07T00:51:34,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694456) roll requested 2024-12-07T00:51:34,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694463 2024-12-07T00:51:34,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694463 2024-12-07T00:51:34,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694463) roll requested 2024-12-07T00:51:34,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694472 2024-12-07T00:51:34,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694472 2024-12-07T00:51:34,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694472) roll requested 2024-12-07T00:51:34,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694481 2024-12-07T00:51:34,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694481 2024-12-07T00:51:34,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694481) roll requested 2024-12-07T00:51:34,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694492 2024-12-07T00:51:34,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694492 2024-12-07T00:51:34,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694492) roll requested 2024-12-07T00:51:34,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694501 2024-12-07T00:51:34,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694501 2024-12-07T00:51:34,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694501) roll requested 2024-12-07T00:51:34,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694511 2024-12-07T00:51:34,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694511 2024-12-07T00:51:34,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694511) roll requested 2024-12-07T00:51:34,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694518 2024-12-07T00:51:34,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694518 2024-12-07T00:51:34,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694518) roll requested 2024-12-07T00:51:34,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694523 2024-12-07T00:51:34,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694523 2024-12-07T00:51:34,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694523) roll requested 2024-12-07T00:51:34,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694529 2024-12-07T00:51:34,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694529 2024-12-07T00:51:34,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694529) roll requested 2024-12-07T00:51:34,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694536 2024-12-07T00:51:34,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694536 2024-12-07T00:51:34,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694536) roll requested 2024-12-07T00:51:34,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694543 2024-12-07T00:51:34,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694543 2024-12-07T00:51:34,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694543) roll requested 2024-12-07T00:51:34,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694551 2024-12-07T00:51:34,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694551 2024-12-07T00:51:34,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694551) roll requested 2024-12-07T00:51:34,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694561 2024-12-07T00:51:34,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694561 2024-12-07T00:51:34,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694561) roll requested 2024-12-07T00:51:34,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694570 2024-12-07T00:51:34,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694570 2024-12-07T00:51:34,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694570) roll requested 2024-12-07T00:51:34,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694579 2024-12-07T00:51:34,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694579 2024-12-07T00:51:34,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694579) roll requested 2024-12-07T00:51:34,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694589 2024-12-07T00:51:34,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694589 2024-12-07T00:51:34,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694589) roll requested 2024-12-07T00:51:34,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694595 2024-12-07T00:51:34,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694595 2024-12-07T00:51:34,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694595) roll requested 2024-12-07T00:51:34,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694601 2024-12-07T00:51:34,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694601 2024-12-07T00:51:34,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694601) roll requested 2024-12-07T00:51:34,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694609 2024-12-07T00:51:34,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694609 2024-12-07T00:51:34,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694609) roll requested 2024-12-07T00:51:34,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694617 2024-12-07T00:51:34,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694617 2024-12-07T00:51:34,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694617) roll requested 2024-12-07T00:51:34,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694624 2024-12-07T00:51:34,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694624 2024-12-07T00:51:34,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694624) roll requested 2024-12-07T00:51:34,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694632 2024-12-07T00:51:34,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694632 2024-12-07T00:51:34,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694632) roll requested 2024-12-07T00:51:34,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694639 2024-12-07T00:51:34,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694639 2024-12-07T00:51:34,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694639) roll requested 2024-12-07T00:51:34,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694648 2024-12-07T00:51:34,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694648 2024-12-07T00:51:34,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694648) roll requested 2024-12-07T00:51:34,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694655 2024-12-07T00:51:34,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694655 2024-12-07T00:51:34,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694655) roll requested 2024-12-07T00:51:34,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694664 2024-12-07T00:51:34,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694664 2024-12-07T00:51:34,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694664) roll requested 2024-12-07T00:51:34,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694669 2024-12-07T00:51:34,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694669 2024-12-07T00:51:34,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694669) roll requested 2024-12-07T00:51:34,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694674 2024-12-07T00:51:34,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694674 2024-12-07T00:51:34,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694674) roll requested 2024-12-07T00:51:34,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694681 2024-12-07T00:51:34,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694681 2024-12-07T00:51:34,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694681) roll requested 2024-12-07T00:51:34,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694688 2024-12-07T00:51:34,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694688 2024-12-07T00:51:34,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694688) roll requested 2024-12-07T00:51:34,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694695 2024-12-07T00:51:34,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694695 2024-12-07T00:51:34,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694695) roll requested 2024-12-07T00:51:34,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694704 2024-12-07T00:51:34,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694704 2024-12-07T00:51:34,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694704) roll requested 2024-12-07T00:51:34,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694712 2024-12-07T00:51:34,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694712 2024-12-07T00:51:34,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694712) roll requested 2024-12-07T00:51:34,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694720 2024-12-07T00:51:34,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694720 2024-12-07T00:51:34,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694720) roll requested 2024-12-07T00:51:34,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694726 2024-12-07T00:51:34,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694726 2024-12-07T00:51:34,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694726) roll requested 2024-12-07T00:51:34,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694731 2024-12-07T00:51:34,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694731 2024-12-07T00:51:34,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694731) roll requested 2024-12-07T00:51:34,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694738 2024-12-07T00:51:34,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694738 2024-12-07T00:51:34,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694738) roll requested 2024-12-07T00:51:34,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694745 2024-12-07T00:51:34,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694745 2024-12-07T00:51:34,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694745) roll requested 2024-12-07T00:51:34,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694752 2024-12-07T00:51:34,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694752 2024-12-07T00:51:34,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694752) roll requested 2024-12-07T00:51:34,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694761 2024-12-07T00:51:34,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694761 2024-12-07T00:51:34,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694761) roll requested 2024-12-07T00:51:34,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694770 2024-12-07T00:51:34,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694770 2024-12-07T00:51:34,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694770) roll requested 2024-12-07T00:51:34,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694780 2024-12-07T00:51:34,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694780 2024-12-07T00:51:34,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694780) roll requested 2024-12-07T00:51:34,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694789 2024-12-07T00:51:34,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694789 2024-12-07T00:51:34,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694789) roll requested 2024-12-07T00:51:34,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694799 2024-12-07T00:51:34,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694799 2024-12-07T00:51:34,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694799) roll requested 2024-12-07T00:51:34,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694807 2024-12-07T00:51:34,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694807 2024-12-07T00:51:34,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694807) roll requested 2024-12-07T00:51:34,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694813 2024-12-07T00:51:34,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694813 2024-12-07T00:51:34,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694813) roll requested 2024-12-07T00:51:34,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694820 2024-12-07T00:51:34,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694820 2024-12-07T00:51:34,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694820) roll requested 2024-12-07T00:51:34,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694828 2024-12-07T00:51:34,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694828 2024-12-07T00:51:34,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694828) roll requested 2024-12-07T00:51:34,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694836 2024-12-07T00:51:34,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694836 2024-12-07T00:51:34,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694836) roll requested 2024-12-07T00:51:34,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694844 2024-12-07T00:51:34,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694844 2024-12-07T00:51:34,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694844) roll requested 2024-12-07T00:51:34,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694851 2024-12-07T00:51:34,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694851 2024-12-07T00:51:34,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694851) roll requested 2024-12-07T00:51:34,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694859 2024-12-07T00:51:34,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694859 2024-12-07T00:51:34,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694859) roll requested 2024-12-07T00:51:34,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694868 2024-12-07T00:51:34,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694868 2024-12-07T00:51:34,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694868) roll requested 2024-12-07T00:51:34,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694876 2024-12-07T00:51:34,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694876 2024-12-07T00:51:34,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694876) roll requested 2024-12-07T00:51:34,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694881 2024-12-07T00:51:34,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694881 2024-12-07T00:51:34,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694881) roll requested 2024-12-07T00:51:34,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694886 2024-12-07T00:51:34,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694886 2024-12-07T00:51:34,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694886) roll requested 2024-12-07T00:51:34,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694893 2024-12-07T00:51:34,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694893 2024-12-07T00:51:34,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694893) roll requested 2024-12-07T00:51:34,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694900 2024-12-07T00:51:34,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694900 2024-12-07T00:51:34,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694900) roll requested 2024-12-07T00:51:34,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694908 2024-12-07T00:51:34,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694908 2024-12-07T00:51:34,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694908) roll requested 2024-12-07T00:51:34,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694914 2024-12-07T00:51:34,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694914 2024-12-07T00:51:34,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694914) roll requested 2024-12-07T00:51:34,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694921 2024-12-07T00:51:34,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694921 2024-12-07T00:51:34,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694921) roll requested 2024-12-07T00:51:34,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694929 2024-12-07T00:51:34,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694929 2024-12-07T00:51:34,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694929) roll requested 2024-12-07T00:51:34,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694937 2024-12-07T00:51:34,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694937 2024-12-07T00:51:34,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694937) roll requested 2024-12-07T00:51:34,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694944 2024-12-07T00:51:34,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694944 2024-12-07T00:51:34,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694944) roll requested 2024-12-07T00:51:34,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694952 2024-12-07T00:51:34,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694952 2024-12-07T00:51:34,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694952) roll requested 2024-12-07T00:51:34,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694961 2024-12-07T00:51:34,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694961 2024-12-07T00:51:34,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694961) roll requested 2024-12-07T00:51:34,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694968 2024-12-07T00:51:34,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694968 2024-12-07T00:51:34,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694968) roll requested 2024-12-07T00:51:34,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694975 2024-12-07T00:51:34,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694975 2024-12-07T00:51:34,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694975) roll requested 2024-12-07T00:51:34,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694982 2024-12-07T00:51:34,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694982 2024-12-07T00:51:34,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694982) roll requested 2024-12-07T00:51:34,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694989 2024-12-07T00:51:34,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:34,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694989 2024-12-07T00:51:34,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:34,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:34,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694989) roll requested 2024-12-07T00:51:34,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532694998 2024-12-07T00:51:35,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694998 2024-12-07T00:51:35,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532694998) roll requested 2024-12-07T00:51:35,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695007 2024-12-07T00:51:35,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532694998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695007 2024-12-07T00:51:35,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695007) roll requested 2024-12-07T00:51:35,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695016 2024-12-07T00:51:35,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695016 2024-12-07T00:51:35,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695016) roll requested 2024-12-07T00:51:35,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695024 2024-12-07T00:51:35,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695024 2024-12-07T00:51:35,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695024) roll requested 2024-12-07T00:51:35,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695033 2024-12-07T00:51:35,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695033 2024-12-07T00:51:35,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695033) roll requested 2024-12-07T00:51:35,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695040 2024-12-07T00:51:35,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695040 2024-12-07T00:51:35,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695040) roll requested 2024-12-07T00:51:35,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695045 2024-12-07T00:51:35,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695045 2024-12-07T00:51:35,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695045) roll requested 2024-12-07T00:51:35,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695051 2024-12-07T00:51:35,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695051 2024-12-07T00:51:35,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695051) roll requested 2024-12-07T00:51:35,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695058 2024-12-07T00:51:35,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695058 2024-12-07T00:51:35,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695058) roll requested 2024-12-07T00:51:35,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695065 2024-12-07T00:51:35,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695065 2024-12-07T00:51:35,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695065) roll requested 2024-12-07T00:51:35,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695073 2024-12-07T00:51:35,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695073 2024-12-07T00:51:35,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695073) roll requested 2024-12-07T00:51:35,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695081 2024-12-07T00:51:35,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695081 2024-12-07T00:51:35,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695081) roll requested 2024-12-07T00:51:35,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695090 2024-12-07T00:51:35,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695090 2024-12-07T00:51:35,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695090) roll requested 2024-12-07T00:51:35,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695099 2024-12-07T00:51:35,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695099 2024-12-07T00:51:35,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695099) roll requested 2024-12-07T00:51:35,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695108 2024-12-07T00:51:35,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695108 2024-12-07T00:51:35,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695108) roll requested 2024-12-07T00:51:35,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695117 2024-12-07T00:51:35,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695117 2024-12-07T00:51:35,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695117) roll requested 2024-12-07T00:51:35,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695126 2024-12-07T00:51:35,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695126 2024-12-07T00:51:35,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695126) roll requested 2024-12-07T00:51:35,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695133 2024-12-07T00:51:35,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695133 2024-12-07T00:51:35,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695133) roll requested 2024-12-07T00:51:35,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695139 2024-12-07T00:51:35,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695139 2024-12-07T00:51:35,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695139) roll requested 2024-12-07T00:51:35,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695147 2024-12-07T00:51:35,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695147 2024-12-07T00:51:35,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695147) roll requested 2024-12-07T00:51:35,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695154 2024-12-07T00:51:35,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695154 2024-12-07T00:51:35,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695154) roll requested 2024-12-07T00:51:35,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695162 2024-12-07T00:51:35,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695162 2024-12-07T00:51:35,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695162) roll requested 2024-12-07T00:51:35,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695170 2024-12-07T00:51:35,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695170 2024-12-07T00:51:35,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695170) roll requested 2024-12-07T00:51:35,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695179 2024-12-07T00:51:35,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695179 2024-12-07T00:51:35,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695179) roll requested 2024-12-07T00:51:35,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695187 2024-12-07T00:51:35,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695187 2024-12-07T00:51:35,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695187) roll requested 2024-12-07T00:51:35,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695195 2024-12-07T00:51:35,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695195 2024-12-07T00:51:35,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695195) roll requested 2024-12-07T00:51:35,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695203 2024-12-07T00:51:35,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695203 2024-12-07T00:51:35,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695203) roll requested 2024-12-07T00:51:35,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695212 2024-12-07T00:51:35,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695212 2024-12-07T00:51:35,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695212) roll requested 2024-12-07T00:51:35,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695220 2024-12-07T00:51:35,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695220 2024-12-07T00:51:35,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695220) roll requested 2024-12-07T00:51:35,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695227 2024-12-07T00:51:35,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695227 2024-12-07T00:51:35,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695227) roll requested 2024-12-07T00:51:35,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695233 2024-12-07T00:51:35,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695233 2024-12-07T00:51:35,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695233) roll requested 2024-12-07T00:51:35,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695237 2024-12-07T00:51:35,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695237 2024-12-07T00:51:35,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695237) roll requested 2024-12-07T00:51:35,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695244 2024-12-07T00:51:35,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695244 2024-12-07T00:51:35,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695244) roll requested 2024-12-07T00:51:35,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695250 2024-12-07T00:51:35,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695250 2024-12-07T00:51:35,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695250) roll requested 2024-12-07T00:51:35,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695258 2024-12-07T00:51:35,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695258 2024-12-07T00:51:35,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695258) roll requested 2024-12-07T00:51:35,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695266 2024-12-07T00:51:35,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695266 2024-12-07T00:51:35,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695266) roll requested 2024-12-07T00:51:35,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695272 2024-12-07T00:51:35,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695272 2024-12-07T00:51:35,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695272) roll requested 2024-12-07T00:51:35,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695278 2024-12-07T00:51:35,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695278 2024-12-07T00:51:35,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695278) roll requested 2024-12-07T00:51:35,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695285 2024-12-07T00:51:35,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695285 2024-12-07T00:51:35,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695285) roll requested 2024-12-07T00:51:35,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695292 2024-12-07T00:51:35,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695292 2024-12-07T00:51:35,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695292) roll requested 2024-12-07T00:51:35,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695301 2024-12-07T00:51:35,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695301 2024-12-07T00:51:35,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695301) roll requested 2024-12-07T00:51:35,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695308 2024-12-07T00:51:35,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695308 2024-12-07T00:51:35,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695308) roll requested 2024-12-07T00:51:35,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695314 2024-12-07T00:51:35,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695314 2024-12-07T00:51:35,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695314) roll requested 2024-12-07T00:51:35,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695322 2024-12-07T00:51:35,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695322 2024-12-07T00:51:35,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695322) roll requested 2024-12-07T00:51:35,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695330 2024-12-07T00:51:35,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695330 2024-12-07T00:51:35,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695330) roll requested 2024-12-07T00:51:35,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695339 2024-12-07T00:51:35,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695339 2024-12-07T00:51:35,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695339) roll requested 2024-12-07T00:51:35,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695350 2024-12-07T00:51:35,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695350 2024-12-07T00:51:35,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695350) roll requested 2024-12-07T00:51:35,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695356 2024-12-07T00:51:35,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695356 2024-12-07T00:51:35,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695356) roll requested 2024-12-07T00:51:35,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695361 2024-12-07T00:51:35,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695361 2024-12-07T00:51:35,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695361) roll requested 2024-12-07T00:51:35,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695368 2024-12-07T00:51:35,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695368 2024-12-07T00:51:35,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695368) roll requested 2024-12-07T00:51:35,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695376 2024-12-07T00:51:35,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695376 2024-12-07T00:51:35,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695376) roll requested 2024-12-07T00:51:35,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695384 2024-12-07T00:51:35,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695384 2024-12-07T00:51:35,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695384) roll requested 2024-12-07T00:51:35,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695393 2024-12-07T00:51:35,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695393 2024-12-07T00:51:35,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695393) roll requested 2024-12-07T00:51:35,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695398 2024-12-07T00:51:35,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695398 2024-12-07T00:51:35,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695398) roll requested 2024-12-07T00:51:35,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695404 2024-12-07T00:51:35,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695404 2024-12-07T00:51:35,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695404) roll requested 2024-12-07T00:51:35,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695411 2024-12-07T00:51:35,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695411 2024-12-07T00:51:35,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695411) roll requested 2024-12-07T00:51:35,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695419 2024-12-07T00:51:35,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695419 2024-12-07T00:51:35,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695419) roll requested 2024-12-07T00:51:35,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695427 2024-12-07T00:51:35,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695427 2024-12-07T00:51:35,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695427) roll requested 2024-12-07T00:51:35,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695435 2024-12-07T00:51:35,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695435 2024-12-07T00:51:35,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695435) roll requested 2024-12-07T00:51:35,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695444 2024-12-07T00:51:35,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695444 2024-12-07T00:51:35,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695444) roll requested 2024-12-07T00:51:35,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695451 2024-12-07T00:51:35,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695451 2024-12-07T00:51:35,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695451) roll requested 2024-12-07T00:51:35,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695458 2024-12-07T00:51:35,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695458 2024-12-07T00:51:35,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695458) roll requested 2024-12-07T00:51:35,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695466 2024-12-07T00:51:35,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695466 2024-12-07T00:51:35,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695466) roll requested 2024-12-07T00:51:35,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695475 2024-12-07T00:51:35,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695475 2024-12-07T00:51:35,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695475) roll requested 2024-12-07T00:51:35,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695482 2024-12-07T00:51:35,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695482 2024-12-07T00:51:35,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695482) roll requested 2024-12-07T00:51:35,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695489 2024-12-07T00:51:35,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695489 2024-12-07T00:51:35,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695489) roll requested 2024-12-07T00:51:35,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695500 2024-12-07T00:51:35,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695500 2024-12-07T00:51:35,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695500) roll requested 2024-12-07T00:51:35,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695507 2024-12-07T00:51:35,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695507 2024-12-07T00:51:35,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695507) roll requested 2024-12-07T00:51:35,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695514 2024-12-07T00:51:35,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695514 2024-12-07T00:51:35,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695514) roll requested 2024-12-07T00:51:35,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695523 2024-12-07T00:51:35,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695523 2024-12-07T00:51:35,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695523) roll requested 2024-12-07T00:51:35,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695531 2024-12-07T00:51:35,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695531 2024-12-07T00:51:35,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695531) roll requested 2024-12-07T00:51:35,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695539 2024-12-07T00:51:35,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695539 2024-12-07T00:51:35,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695539) roll requested 2024-12-07T00:51:35,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695546 2024-12-07T00:51:35,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695546 2024-12-07T00:51:35,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695546) roll requested 2024-12-07T00:51:35,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695556 2024-12-07T00:51:35,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695556 2024-12-07T00:51:35,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695556) roll requested 2024-12-07T00:51:35,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695560 2024-12-07T00:51:35,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695560 2024-12-07T00:51:35,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695560) roll requested 2024-12-07T00:51:35,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695565 2024-12-07T00:51:35,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695565 2024-12-07T00:51:35,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695565) roll requested 2024-12-07T00:51:35,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695571 2024-12-07T00:51:35,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695571 2024-12-07T00:51:35,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695571) roll requested 2024-12-07T00:51:35,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695578 2024-12-07T00:51:35,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695578 2024-12-07T00:51:35,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695578) roll requested 2024-12-07T00:51:35,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695585 2024-12-07T00:51:35,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695585 2024-12-07T00:51:35,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695585) roll requested 2024-12-07T00:51:35,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695592 2024-12-07T00:51:35,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695592 2024-12-07T00:51:35,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695592) roll requested 2024-12-07T00:51:35,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695599 2024-12-07T00:51:35,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695599 2024-12-07T00:51:35,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695599) roll requested 2024-12-07T00:51:35,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695607 2024-12-07T00:51:35,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695607 2024-12-07T00:51:35,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695607) roll requested 2024-12-07T00:51:35,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695614 2024-12-07T00:51:35,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695614 2024-12-07T00:51:35,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695614) roll requested 2024-12-07T00:51:35,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695623 2024-12-07T00:51:35,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695623 2024-12-07T00:51:35,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695623) roll requested 2024-12-07T00:51:35,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695632 2024-12-07T00:51:35,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695632 2024-12-07T00:51:35,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695632) roll requested 2024-12-07T00:51:35,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695641 2024-12-07T00:51:35,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695641 2024-12-07T00:51:35,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695641) roll requested 2024-12-07T00:51:35,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695651 2024-12-07T00:51:35,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695651 2024-12-07T00:51:35,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695651) roll requested 2024-12-07T00:51:35,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695659 2024-12-07T00:51:35,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695659 2024-12-07T00:51:35,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695659) roll requested 2024-12-07T00:51:35,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695667 2024-12-07T00:51:35,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695667 2024-12-07T00:51:35,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695667) roll requested 2024-12-07T00:51:35,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695675 2024-12-07T00:51:35,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695675 2024-12-07T00:51:35,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695675) roll requested 2024-12-07T00:51:35,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695681 2024-12-07T00:51:35,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695681 2024-12-07T00:51:35,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695681) roll requested 2024-12-07T00:51:35,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695689 2024-12-07T00:51:35,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695689 2024-12-07T00:51:35,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695689) roll requested 2024-12-07T00:51:35,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695697 2024-12-07T00:51:35,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695697 2024-12-07T00:51:35,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695697) roll requested 2024-12-07T00:51:35,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695708 2024-12-07T00:51:35,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695708 2024-12-07T00:51:35,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695708) roll requested 2024-12-07T00:51:35,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695715 2024-12-07T00:51:35,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695715 2024-12-07T00:51:35,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695715) roll requested 2024-12-07T00:51:35,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695723 2024-12-07T00:51:35,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695723 2024-12-07T00:51:35,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695723) roll requested 2024-12-07T00:51:35,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695732 2024-12-07T00:51:35,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695732 2024-12-07T00:51:35,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695732) roll requested 2024-12-07T00:51:35,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695738 2024-12-07T00:51:35,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695738 2024-12-07T00:51:35,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695738) roll requested 2024-12-07T00:51:35,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695744 2024-12-07T00:51:35,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695744 2024-12-07T00:51:35,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695744) roll requested 2024-12-07T00:51:35,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695753 2024-12-07T00:51:35,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695753 2024-12-07T00:51:35,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695753) roll requested 2024-12-07T00:51:35,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695765 2024-12-07T00:51:35,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695765 2024-12-07T00:51:35,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695765) roll requested 2024-12-07T00:51:35,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695771 2024-12-07T00:51:35,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695771 2024-12-07T00:51:35,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695771) roll requested 2024-12-07T00:51:35,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695780 2024-12-07T00:51:35,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695780 2024-12-07T00:51:35,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695780) roll requested 2024-12-07T00:51:35,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695785 2024-12-07T00:51:35,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695785 2024-12-07T00:51:35,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695785) roll requested 2024-12-07T00:51:35,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695790 2024-12-07T00:51:35,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695790 2024-12-07T00:51:35,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695790) roll requested 2024-12-07T00:51:35,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695796 2024-12-07T00:51:35,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695796 2024-12-07T00:51:35,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695796) roll requested 2024-12-07T00:51:35,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695802 2024-12-07T00:51:35,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695802 2024-12-07T00:51:35,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695802) roll requested 2024-12-07T00:51:35,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695809 2024-12-07T00:51:35,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695809 2024-12-07T00:51:35,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695809) roll requested 2024-12-07T00:51:35,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695818 2024-12-07T00:51:35,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695818 2024-12-07T00:51:35,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695818) roll requested 2024-12-07T00:51:35,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695827 2024-12-07T00:51:35,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695827 2024-12-07T00:51:35,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695827) roll requested 2024-12-07T00:51:35,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695836 2024-12-07T00:51:35,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695836 2024-12-07T00:51:35,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695836) roll requested 2024-12-07T00:51:35,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695840 2024-12-07T00:51:35,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695840 2024-12-07T00:51:35,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695840) roll requested 2024-12-07T00:51:35,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695845 2024-12-07T00:51:35,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695845 2024-12-07T00:51:35,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695845) roll requested 2024-12-07T00:51:35,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695852 2024-12-07T00:51:35,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695852 2024-12-07T00:51:35,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695852) roll requested 2024-12-07T00:51:35,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695859 2024-12-07T00:51:35,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695859 2024-12-07T00:51:35,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695859) roll requested 2024-12-07T00:51:35,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695869 2024-12-07T00:51:35,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695869 2024-12-07T00:51:35,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695869) roll requested 2024-12-07T00:51:35,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695878 2024-12-07T00:51:35,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695878 2024-12-07T00:51:35,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695878) roll requested 2024-12-07T00:51:35,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695883 2024-12-07T00:51:35,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695883 2024-12-07T00:51:35,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695883) roll requested 2024-12-07T00:51:35,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695888 2024-12-07T00:51:35,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695888 2024-12-07T00:51:35,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695888) roll requested 2024-12-07T00:51:35,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695895 2024-12-07T00:51:35,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695895 2024-12-07T00:51:35,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695895) roll requested 2024-12-07T00:51:35,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695901 2024-12-07T00:51:35,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695901 2024-12-07T00:51:35,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695901) roll requested 2024-12-07T00:51:35,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695910 2024-12-07T00:51:35,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695910 2024-12-07T00:51:35,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695910) roll requested 2024-12-07T00:51:35,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695918 2024-12-07T00:51:35,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695918 2024-12-07T00:51:35,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695918) roll requested 2024-12-07T00:51:35,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695927 2024-12-07T00:51:35,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695927 2024-12-07T00:51:35,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695927) roll requested 2024-12-07T00:51:35,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695937 2024-12-07T00:51:35,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695937 2024-12-07T00:51:35,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695937) roll requested 2024-12-07T00:51:35,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695948 2024-12-07T00:51:35,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695948 2024-12-07T00:51:35,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695948) roll requested 2024-12-07T00:51:35,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695955 2024-12-07T00:51:35,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695955 2024-12-07T00:51:35,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695955) roll requested 2024-12-07T00:51:35,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695960 2024-12-07T00:51:35,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695960 2024-12-07T00:51:35,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695960) roll requested 2024-12-07T00:51:35,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695965 2024-12-07T00:51:35,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695965 2024-12-07T00:51:35,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695965) roll requested 2024-12-07T00:51:35,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695971 2024-12-07T00:51:35,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695971 2024-12-07T00:51:35,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695971) roll requested 2024-12-07T00:51:35,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695978 2024-12-07T00:51:35,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695978 2024-12-07T00:51:35,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695978) roll requested 2024-12-07T00:51:35,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695986 2024-12-07T00:51:35,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:35,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695986 2024-12-07T00:51:35,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:35,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:35,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695986) roll requested 2024-12-07T00:51:35,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532695995 2024-12-07T00:51:36,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695995 2024-12-07T00:51:36,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532695995) roll requested 2024-12-07T00:51:36,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696004 2024-12-07T00:51:36,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532695995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696004 2024-12-07T00:51:36,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696004) roll requested 2024-12-07T00:51:36,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696014 2024-12-07T00:51:36,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696014 2024-12-07T00:51:36,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696014) roll requested 2024-12-07T00:51:36,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696025 2024-12-07T00:51:36,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696025 2024-12-07T00:51:36,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696025) roll requested 2024-12-07T00:51:36,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696034 2024-12-07T00:51:36,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696034 2024-12-07T00:51:36,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696034) roll requested 2024-12-07T00:51:36,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696040 2024-12-07T00:51:36,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696040 2024-12-07T00:51:36,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696040) roll requested 2024-12-07T00:51:36,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696045 2024-12-07T00:51:36,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696045 2024-12-07T00:51:36,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696045) roll requested 2024-12-07T00:51:36,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696051 2024-12-07T00:51:36,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696051 2024-12-07T00:51:36,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696051) roll requested 2024-12-07T00:51:36,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696057 2024-12-07T00:51:36,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696057 2024-12-07T00:51:36,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696057) roll requested 2024-12-07T00:51:36,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696064 2024-12-07T00:51:36,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696064 2024-12-07T00:51:36,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696064) roll requested 2024-12-07T00:51:36,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696072 2024-12-07T00:51:36,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696072 2024-12-07T00:51:36,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696072) roll requested 2024-12-07T00:51:36,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696080 2024-12-07T00:51:36,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696080 2024-12-07T00:51:36,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696080) roll requested 2024-12-07T00:51:36,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696088 2024-12-07T00:51:36,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696088 2024-12-07T00:51:36,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696088) roll requested 2024-12-07T00:51:36,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696094 2024-12-07T00:51:36,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696094 2024-12-07T00:51:36,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696094) roll requested 2024-12-07T00:51:36,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696100 2024-12-07T00:51:36,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696100 2024-12-07T00:51:36,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696100) roll requested 2024-12-07T00:51:36,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696105 2024-12-07T00:51:36,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696105 2024-12-07T00:51:36,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696105) roll requested 2024-12-07T00:51:36,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696110 2024-12-07T00:51:36,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696110 2024-12-07T00:51:36,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696110) roll requested 2024-12-07T00:51:36,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696116 2024-12-07T00:51:36,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696116 2024-12-07T00:51:36,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696116) roll requested 2024-12-07T00:51:36,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696121 2024-12-07T00:51:36,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696121 2024-12-07T00:51:36,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696121) roll requested 2024-12-07T00:51:36,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696126 2024-12-07T00:51:36,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696126 2024-12-07T00:51:36,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696126) roll requested 2024-12-07T00:51:36,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696131 2024-12-07T00:51:36,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696131 2024-12-07T00:51:36,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696131) roll requested 2024-12-07T00:51:36,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696136 2024-12-07T00:51:36,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696136 2024-12-07T00:51:36,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696136) roll requested 2024-12-07T00:51:36,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696142 2024-12-07T00:51:36,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696142 2024-12-07T00:51:36,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696142) roll requested 2024-12-07T00:51:36,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696150 2024-12-07T00:51:36,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696150 2024-12-07T00:51:36,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696150) roll requested 2024-12-07T00:51:36,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696158 2024-12-07T00:51:36,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696158 2024-12-07T00:51:36,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696158) roll requested 2024-12-07T00:51:36,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696169 2024-12-07T00:51:36,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696169 2024-12-07T00:51:36,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696169) roll requested 2024-12-07T00:51:36,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696179 2024-12-07T00:51:36,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696179 2024-12-07T00:51:36,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696179) roll requested 2024-12-07T00:51:36,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696190 2024-12-07T00:51:36,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696190 2024-12-07T00:51:36,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696190) roll requested 2024-12-07T00:51:36,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696198 2024-12-07T00:51:36,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696198 2024-12-07T00:51:36,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696198) roll requested 2024-12-07T00:51:36,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696206 2024-12-07T00:51:36,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696206 2024-12-07T00:51:36,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696206) roll requested 2024-12-07T00:51:36,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696212 2024-12-07T00:51:36,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696212 2024-12-07T00:51:36,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696212) roll requested 2024-12-07T00:51:36,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696218 2024-12-07T00:51:36,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696218 2024-12-07T00:51:36,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696218) roll requested 2024-12-07T00:51:36,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696223 2024-12-07T00:51:36,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696223 2024-12-07T00:51:36,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696223) roll requested 2024-12-07T00:51:36,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696228 2024-12-07T00:51:36,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696228 2024-12-07T00:51:36,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696228) roll requested 2024-12-07T00:51:36,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696233 2024-12-07T00:51:36,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696233 2024-12-07T00:51:36,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696233) roll requested 2024-12-07T00:51:36,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696240 2024-12-07T00:51:36,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696240 2024-12-07T00:51:36,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696240) roll requested 2024-12-07T00:51:36,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696247 2024-12-07T00:51:36,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696247 2024-12-07T00:51:36,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696247) roll requested 2024-12-07T00:51:36,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696254 2024-12-07T00:51:36,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696254 2024-12-07T00:51:36,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696254) roll requested 2024-12-07T00:51:36,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696261 2024-12-07T00:51:36,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696261 2024-12-07T00:51:36,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696261) roll requested 2024-12-07T00:51:36,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696269 2024-12-07T00:51:36,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696269 2024-12-07T00:51:36,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696269) roll requested 2024-12-07T00:51:36,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696279 2024-12-07T00:51:36,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696279 2024-12-07T00:51:36,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:36,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696279) roll requested 2024-12-07T00:51:36,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696289 2024-12-07T00:51:36,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696289 2024-12-07T00:51:36,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696289) roll requested 2024-12-07T00:51:36,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696295 2024-12-07T00:51:36,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696295 2024-12-07T00:51:36,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696295) roll requested 2024-12-07T00:51:36,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696301 2024-12-07T00:51:36,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696301 2024-12-07T00:51:36,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696301) roll requested 2024-12-07T00:51:36,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696309 2024-12-07T00:51:36,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696309 2024-12-07T00:51:36,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696309) roll requested 2024-12-07T00:51:36,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696316 2024-12-07T00:51:36,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696316 2024-12-07T00:51:36,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696316) roll requested 2024-12-07T00:51:36,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696324 2024-12-07T00:51:36,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696324 2024-12-07T00:51:36,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696324) roll requested 2024-12-07T00:51:36,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696332 2024-12-07T00:51:36,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696332 2024-12-07T00:51:36,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696332) roll requested 2024-12-07T00:51:36,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696339 2024-12-07T00:51:36,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696339 2024-12-07T00:51:36,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696339) roll requested 2024-12-07T00:51:36,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696345 2024-12-07T00:51:36,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696345 2024-12-07T00:51:36,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696345) roll requested 2024-12-07T00:51:36,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696353 2024-12-07T00:51:36,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696353 2024-12-07T00:51:36,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696353) roll requested 2024-12-07T00:51:36,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696361 2024-12-07T00:51:36,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696361 2024-12-07T00:51:36,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696361) roll requested 2024-12-07T00:51:36,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696366 2024-12-07T00:51:36,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696366 2024-12-07T00:51:36,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696366) roll requested 2024-12-07T00:51:36,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696371 2024-12-07T00:51:36,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696371 2024-12-07T00:51:36,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696371) roll requested 2024-12-07T00:51:36,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696378 2024-12-07T00:51:36,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696378 2024-12-07T00:51:36,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696378) roll requested 2024-12-07T00:51:36,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696388 2024-12-07T00:51:36,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696388 2024-12-07T00:51:36,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696388) roll requested 2024-12-07T00:51:36,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696397 2024-12-07T00:51:36,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696397 2024-12-07T00:51:36,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696397) roll requested 2024-12-07T00:51:36,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696402 2024-12-07T00:51:36,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696402 2024-12-07T00:51:36,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696402) roll requested 2024-12-07T00:51:36,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696408 2024-12-07T00:51:36,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696408 2024-12-07T00:51:36,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696408) roll requested 2024-12-07T00:51:36,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696417 2024-12-07T00:51:36,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696417 2024-12-07T00:51:36,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696417) roll requested 2024-12-07T00:51:36,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696423 2024-12-07T00:51:36,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696423 2024-12-07T00:51:36,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696423) roll requested 2024-12-07T00:51:36,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696429 2024-12-07T00:51:36,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696429 2024-12-07T00:51:36,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696429) roll requested 2024-12-07T00:51:36,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696434 2024-12-07T00:51:36,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696434 2024-12-07T00:51:36,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696434) roll requested 2024-12-07T00:51:36,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696440 2024-12-07T00:51:36,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696440 2024-12-07T00:51:36,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696440) roll requested 2024-12-07T00:51:36,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696447 2024-12-07T00:51:36,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696447 2024-12-07T00:51:36,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696447) roll requested 2024-12-07T00:51:36,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696455 2024-12-07T00:51:36,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696455 2024-12-07T00:51:36,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696455) roll requested 2024-12-07T00:51:36,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696463 2024-12-07T00:51:36,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696463 2024-12-07T00:51:36,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696463) roll requested 2024-12-07T00:51:36,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696472 2024-12-07T00:51:36,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696472 2024-12-07T00:51:36,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696472) roll requested 2024-12-07T00:51:36,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696479 2024-12-07T00:51:36,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696479 2024-12-07T00:51:36,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696479) roll requested 2024-12-07T00:51:36,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696488 2024-12-07T00:51:36,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696488 2024-12-07T00:51:36,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696488) roll requested 2024-12-07T00:51:36,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696496 2024-12-07T00:51:36,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696496 2024-12-07T00:51:36,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696496) roll requested 2024-12-07T00:51:36,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696505 2024-12-07T00:51:36,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696505 2024-12-07T00:51:36,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696505) roll requested 2024-12-07T00:51:36,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696514 2024-12-07T00:51:36,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696514 2024-12-07T00:51:36,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696514) roll requested 2024-12-07T00:51:36,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696521 2024-12-07T00:51:36,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696521 2024-12-07T00:51:36,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696521) roll requested 2024-12-07T00:51:36,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696529 2024-12-07T00:51:36,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696529 2024-12-07T00:51:36,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696529) roll requested 2024-12-07T00:51:36,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696534 2024-12-07T00:51:36,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696534 2024-12-07T00:51:36,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696534) roll requested 2024-12-07T00:51:36,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696539 2024-12-07T00:51:36,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696539 2024-12-07T00:51:36,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696539) roll requested 2024-12-07T00:51:36,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696547 2024-12-07T00:51:36,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696547 2024-12-07T00:51:36,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696547) roll requested 2024-12-07T00:51:36,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696554 2024-12-07T00:51:36,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696554 2024-12-07T00:51:36,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696554) roll requested 2024-12-07T00:51:36,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696562 2024-12-07T00:51:36,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696562 2024-12-07T00:51:36,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696562) roll requested 2024-12-07T00:51:36,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696571 2024-12-07T00:51:36,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696571 2024-12-07T00:51:36,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696571) roll requested 2024-12-07T00:51:36,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696580 2024-12-07T00:51:36,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696580 2024-12-07T00:51:36,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696580) roll requested 2024-12-07T00:51:36,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696587 2024-12-07T00:51:36,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696587 2024-12-07T00:51:36,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696587) roll requested 2024-12-07T00:51:36,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696597 2024-12-07T00:51:36,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696597 2024-12-07T00:51:36,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696597) roll requested 2024-12-07T00:51:36,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696606 2024-12-07T00:51:36,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696606 2024-12-07T00:51:36,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696606) roll requested 2024-12-07T00:51:36,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696615 2024-12-07T00:51:36,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696615 2024-12-07T00:51:36,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696615) roll requested 2024-12-07T00:51:36,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696623 2024-12-07T00:51:36,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696623 2024-12-07T00:51:36,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696623) roll requested 2024-12-07T00:51:36,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696631 2024-12-07T00:51:36,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696631 2024-12-07T00:51:36,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696631) roll requested 2024-12-07T00:51:36,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696640 2024-12-07T00:51:36,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696640 2024-12-07T00:51:36,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696640) roll requested 2024-12-07T00:51:36,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696648 2024-12-07T00:51:36,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696648 2024-12-07T00:51:36,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696648) roll requested 2024-12-07T00:51:36,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696655 2024-12-07T00:51:36,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696655 2024-12-07T00:51:36,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696655) roll requested 2024-12-07T00:51:36,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696662 2024-12-07T00:51:36,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696662 2024-12-07T00:51:36,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696662) roll requested 2024-12-07T00:51:36,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696671 2024-12-07T00:51:36,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696671 2024-12-07T00:51:36,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696671) roll requested 2024-12-07T00:51:36,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696680 2024-12-07T00:51:36,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696680 2024-12-07T00:51:36,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696680) roll requested 2024-12-07T00:51:36,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696689 2024-12-07T00:51:36,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696689 2024-12-07T00:51:36,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696689) roll requested 2024-12-07T00:51:36,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696697 2024-12-07T00:51:36,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696697 2024-12-07T00:51:36,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696697) roll requested 2024-12-07T00:51:36,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696706 2024-12-07T00:51:36,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696706 2024-12-07T00:51:36,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696706) roll requested 2024-12-07T00:51:36,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696718 2024-12-07T00:51:36,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696718 2024-12-07T00:51:36,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696718) roll requested 2024-12-07T00:51:36,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696726 2024-12-07T00:51:36,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696726 2024-12-07T00:51:36,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696726) roll requested 2024-12-07T00:51:36,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696736 2024-12-07T00:51:36,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696736 2024-12-07T00:51:36,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696736) roll requested 2024-12-07T00:51:36,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696744 2024-12-07T00:51:36,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696744 2024-12-07T00:51:36,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696744) roll requested 2024-12-07T00:51:36,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696753 2024-12-07T00:51:36,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696753 2024-12-07T00:51:36,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696753) roll requested 2024-12-07T00:51:36,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696763 2024-12-07T00:51:36,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696763 2024-12-07T00:51:36,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696763) roll requested 2024-12-07T00:51:36,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696772 2024-12-07T00:51:36,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696772 2024-12-07T00:51:36,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696772) roll requested 2024-12-07T00:51:36,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696782 2024-12-07T00:51:36,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696782 2024-12-07T00:51:36,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696782) roll requested 2024-12-07T00:51:36,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696791 2024-12-07T00:51:36,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696791 2024-12-07T00:51:36,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696791) roll requested 2024-12-07T00:51:36,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696800 2024-12-07T00:51:36,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696800 2024-12-07T00:51:36,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696800) roll requested 2024-12-07T00:51:36,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696809 2024-12-07T00:51:36,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696809 2024-12-07T00:51:36,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696809) roll requested 2024-12-07T00:51:36,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696815 2024-12-07T00:51:36,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696815 2024-12-07T00:51:36,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696815) roll requested 2024-12-07T00:51:36,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696821 2024-12-07T00:51:36,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696821 2024-12-07T00:51:36,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696821) roll requested 2024-12-07T00:51:36,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696829 2024-12-07T00:51:36,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696829 2024-12-07T00:51:36,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696829) roll requested 2024-12-07T00:51:36,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696838 2024-12-07T00:51:36,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696838 2024-12-07T00:51:36,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696838) roll requested 2024-12-07T00:51:36,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696845 2024-12-07T00:51:36,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696845 2024-12-07T00:51:36,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696845) roll requested 2024-12-07T00:51:36,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696850 2024-12-07T00:51:36,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696850 2024-12-07T00:51:36,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696850) roll requested 2024-12-07T00:51:36,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696856 2024-12-07T00:51:36,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696856 2024-12-07T00:51:36,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696856) roll requested 2024-12-07T00:51:36,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696861 2024-12-07T00:51:36,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696861 2024-12-07T00:51:36,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696861) roll requested 2024-12-07T00:51:36,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696866 2024-12-07T00:51:36,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696866 2024-12-07T00:51:36,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696866) roll requested 2024-12-07T00:51:36,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696872 2024-12-07T00:51:36,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696872 2024-12-07T00:51:36,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696872) roll requested 2024-12-07T00:51:36,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696880 2024-12-07T00:51:36,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696880 2024-12-07T00:51:36,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696880) roll requested 2024-12-07T00:51:36,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696886 2024-12-07T00:51:36,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696886 2024-12-07T00:51:36,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696886) roll requested 2024-12-07T00:51:36,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696892 2024-12-07T00:51:36,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696892 2024-12-07T00:51:36,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696892) roll requested 2024-12-07T00:51:36,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696901 2024-12-07T00:51:36,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696901 2024-12-07T00:51:36,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696901) roll requested 2024-12-07T00:51:36,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696906 2024-12-07T00:51:36,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696906 2024-12-07T00:51:36,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696906) roll requested 2024-12-07T00:51:36,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696913 2024-12-07T00:51:36,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696913 2024-12-07T00:51:36,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696913) roll requested 2024-12-07T00:51:36,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696919 2024-12-07T00:51:36,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696919 2024-12-07T00:51:36,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696919) roll requested 2024-12-07T00:51:36,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696925 2024-12-07T00:51:36,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696925 2024-12-07T00:51:36,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696925) roll requested 2024-12-07T00:51:36,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696930 2024-12-07T00:51:36,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696930 2024-12-07T00:51:36,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696930) roll requested 2024-12-07T00:51:36,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696937 2024-12-07T00:51:36,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696937 2024-12-07T00:51:36,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696937) roll requested 2024-12-07T00:51:36,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696944 2024-12-07T00:51:36,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696944 2024-12-07T00:51:36,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696944) roll requested 2024-12-07T00:51:36,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696952 2024-12-07T00:51:36,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696952 2024-12-07T00:51:36,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696952) roll requested 2024-12-07T00:51:36,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696961 2024-12-07T00:51:36,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696961 2024-12-07T00:51:36,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696961) roll requested 2024-12-07T00:51:36,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696967 2024-12-07T00:51:36,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696967 2024-12-07T00:51:36,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696967) roll requested 2024-12-07T00:51:36,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696974 2024-12-07T00:51:36,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696974 2024-12-07T00:51:36,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696974) roll requested 2024-12-07T00:51:36,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696979 2024-12-07T00:51:36,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696979 2024-12-07T00:51:36,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696979) roll requested 2024-12-07T00:51:36,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696984 2024-12-07T00:51:36,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696984 2024-12-07T00:51:36,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696984) roll requested 2024-12-07T00:51:36,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696989 2024-12-07T00:51:36,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:36,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696989 2024-12-07T00:51:36,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:36,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:36,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:36,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:36,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696989) roll requested 2024-12-07T00:51:36,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532696995 2024-12-07T00:51:36,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696995 2024-12-07T00:51:37,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532696995) roll requested 2024-12-07T00:51:37,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697001 2024-12-07T00:51:37,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532696995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697001 2024-12-07T00:51:37,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697001) roll requested 2024-12-07T00:51:37,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697009 2024-12-07T00:51:37,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697009 2024-12-07T00:51:37,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697009) roll requested 2024-12-07T00:51:37,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697014 2024-12-07T00:51:37,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697014 2024-12-07T00:51:37,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697014) roll requested 2024-12-07T00:51:37,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697019 2024-12-07T00:51:37,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697019 2024-12-07T00:51:37,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697019) roll requested 2024-12-07T00:51:37,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697024 2024-12-07T00:51:37,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697024 2024-12-07T00:51:37,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697024) roll requested 2024-12-07T00:51:37,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697029 2024-12-07T00:51:37,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697029 2024-12-07T00:51:37,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697029) roll requested 2024-12-07T00:51:37,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697035 2024-12-07T00:51:37,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697035 2024-12-07T00:51:37,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697035) roll requested 2024-12-07T00:51:37,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697042 2024-12-07T00:51:37,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697042 2024-12-07T00:51:37,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697042) roll requested 2024-12-07T00:51:37,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697048 2024-12-07T00:51:37,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697048 2024-12-07T00:51:37,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697048) roll requested 2024-12-07T00:51:37,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697057 2024-12-07T00:51:37,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697057 2024-12-07T00:51:37,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697057) roll requested 2024-12-07T00:51:37,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697065 2024-12-07T00:51:37,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697065 2024-12-07T00:51:37,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697065) roll requested 2024-12-07T00:51:37,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697074 2024-12-07T00:51:37,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697074 2024-12-07T00:51:37,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697074) roll requested 2024-12-07T00:51:37,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697083 2024-12-07T00:51:37,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697083 2024-12-07T00:51:37,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697083) roll requested 2024-12-07T00:51:37,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697092 2024-12-07T00:51:37,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697092 2024-12-07T00:51:37,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697092) roll requested 2024-12-07T00:51:37,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697103 2024-12-07T00:51:37,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697103 2024-12-07T00:51:37,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697103) roll requested 2024-12-07T00:51:37,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697112 2024-12-07T00:51:37,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697112 2024-12-07T00:51:37,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697112) roll requested 2024-12-07T00:51:37,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697120 2024-12-07T00:51:37,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697120 2024-12-07T00:51:37,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697120) roll requested 2024-12-07T00:51:37,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697129 2024-12-07T00:51:37,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697129 2024-12-07T00:51:37,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697129) roll requested 2024-12-07T00:51:37,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697139 2024-12-07T00:51:37,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697139 2024-12-07T00:51:37,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697139) roll requested 2024-12-07T00:51:37,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697148 2024-12-07T00:51:37,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697148 2024-12-07T00:51:37,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697148) roll requested 2024-12-07T00:51:37,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697155 2024-12-07T00:51:37,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697155 2024-12-07T00:51:37,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697155) roll requested 2024-12-07T00:51:37,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697161 2024-12-07T00:51:37,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697161 2024-12-07T00:51:37,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697161) roll requested 2024-12-07T00:51:37,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697166 2024-12-07T00:51:37,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697166 2024-12-07T00:51:37,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697166) roll requested 2024-12-07T00:51:37,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697173 2024-12-07T00:51:37,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697173 2024-12-07T00:51:37,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697173) roll requested 2024-12-07T00:51:37,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697181 2024-12-07T00:51:37,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697181 2024-12-07T00:51:37,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697181) roll requested 2024-12-07T00:51:37,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697188 2024-12-07T00:51:37,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697188 2024-12-07T00:51:37,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697188) roll requested 2024-12-07T00:51:37,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697196 2024-12-07T00:51:37,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697196 2024-12-07T00:51:37,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697196) roll requested 2024-12-07T00:51:37,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697203 2024-12-07T00:51:37,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697203 2024-12-07T00:51:37,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697203) roll requested 2024-12-07T00:51:37,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697208 2024-12-07T00:51:37,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697208 2024-12-07T00:51:37,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697208) roll requested 2024-12-07T00:51:37,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697213 2024-12-07T00:51:37,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697213 2024-12-07T00:51:37,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697213) roll requested 2024-12-07T00:51:37,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697219 2024-12-07T00:51:37,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697219 2024-12-07T00:51:37,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697219) roll requested 2024-12-07T00:51:37,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697225 2024-12-07T00:51:37,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697225 2024-12-07T00:51:37,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697225) roll requested 2024-12-07T00:51:37,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697230 2024-12-07T00:51:37,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697230 2024-12-07T00:51:37,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697230) roll requested 2024-12-07T00:51:37,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697236 2024-12-07T00:51:37,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697236 2024-12-07T00:51:37,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697236) roll requested 2024-12-07T00:51:37,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697241 2024-12-07T00:51:37,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697241 2024-12-07T00:51:37,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697241) roll requested 2024-12-07T00:51:37,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697248 2024-12-07T00:51:37,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697248 2024-12-07T00:51:37,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697248) roll requested 2024-12-07T00:51:37,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697257 2024-12-07T00:51:37,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697257 2024-12-07T00:51:37,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697257) roll requested 2024-12-07T00:51:37,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697263 2024-12-07T00:51:37,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697263 2024-12-07T00:51:37,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697263) roll requested 2024-12-07T00:51:37,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697269 2024-12-07T00:51:37,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697269 2024-12-07T00:51:37,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697269) roll requested 2024-12-07T00:51:37,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697277 2024-12-07T00:51:37,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697277 2024-12-07T00:51:37,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697277) roll requested 2024-12-07T00:51:37,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697282 2024-12-07T00:51:37,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697282 2024-12-07T00:51:37,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697282) roll requested 2024-12-07T00:51:37,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697288 2024-12-07T00:51:37,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697288 2024-12-07T00:51:37,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697288) roll requested 2024-12-07T00:51:37,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697293 2024-12-07T00:51:37,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697293 2024-12-07T00:51:37,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697293) roll requested 2024-12-07T00:51:37,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697301 2024-12-07T00:51:37,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697301 2024-12-07T00:51:37,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697301) roll requested 2024-12-07T00:51:37,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697309 2024-12-07T00:51:37,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697309 2024-12-07T00:51:37,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697309) roll requested 2024-12-07T00:51:37,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697315 2024-12-07T00:51:37,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697315 2024-12-07T00:51:37,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697315) roll requested 2024-12-07T00:51:37,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697321 2024-12-07T00:51:37,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697321 2024-12-07T00:51:37,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697321) roll requested 2024-12-07T00:51:37,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697328 2024-12-07T00:51:37,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697328 2024-12-07T00:51:37,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697328) roll requested 2024-12-07T00:51:37,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697334 2024-12-07T00:51:37,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697334 2024-12-07T00:51:37,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697334) roll requested 2024-12-07T00:51:37,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697339 2024-12-07T00:51:37,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697339 2024-12-07T00:51:37,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697339) roll requested 2024-12-07T00:51:37,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697344 2024-12-07T00:51:37,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697344 2024-12-07T00:51:37,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697344) roll requested 2024-12-07T00:51:37,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697350 2024-12-07T00:51:37,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697350 2024-12-07T00:51:37,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697350) roll requested 2024-12-07T00:51:37,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697356 2024-12-07T00:51:37,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697356 2024-12-07T00:51:37,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697356) roll requested 2024-12-07T00:51:37,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697361 2024-12-07T00:51:37,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697361 2024-12-07T00:51:37,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697361) roll requested 2024-12-07T00:51:37,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697367 2024-12-07T00:51:37,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697367 2024-12-07T00:51:37,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697367) roll requested 2024-12-07T00:51:37,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697374 2024-12-07T00:51:37,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697374 2024-12-07T00:51:37,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697374) roll requested 2024-12-07T00:51:37,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697382 2024-12-07T00:51:37,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697382 2024-12-07T00:51:37,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697382) roll requested 2024-12-07T00:51:37,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697389 2024-12-07T00:51:37,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697389 2024-12-07T00:51:37,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697389) roll requested 2024-12-07T00:51:37,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697395 2024-12-07T00:51:37,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697395 2024-12-07T00:51:37,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697395) roll requested 2024-12-07T00:51:37,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697401 2024-12-07T00:51:37,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697401 2024-12-07T00:51:37,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697401) roll requested 2024-12-07T00:51:37,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697408 2024-12-07T00:51:37,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697408 2024-12-07T00:51:37,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697408) roll requested 2024-12-07T00:51:37,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697415 2024-12-07T00:51:37,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697415 2024-12-07T00:51:37,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697415) roll requested 2024-12-07T00:51:37,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697424 2024-12-07T00:51:37,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697424 2024-12-07T00:51:37,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697424) roll requested 2024-12-07T00:51:37,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697431 2024-12-07T00:51:37,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697431 2024-12-07T00:51:37,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697431) roll requested 2024-12-07T00:51:37,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697440 2024-12-07T00:51:37,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697440 2024-12-07T00:51:37,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697440) roll requested 2024-12-07T00:51:37,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697449 2024-12-07T00:51:37,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697449 2024-12-07T00:51:37,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697449) roll requested 2024-12-07T00:51:37,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697455 2024-12-07T00:51:37,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697455 2024-12-07T00:51:37,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697455) roll requested 2024-12-07T00:51:37,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697461 2024-12-07T00:51:37,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697461 2024-12-07T00:51:37,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697461) roll requested 2024-12-07T00:51:37,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697467 2024-12-07T00:51:37,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697467 2024-12-07T00:51:37,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697467) roll requested 2024-12-07T00:51:37,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697474 2024-12-07T00:51:37,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697474 2024-12-07T00:51:37,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697474) roll requested 2024-12-07T00:51:37,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697483 2024-12-07T00:51:37,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697483 2024-12-07T00:51:37,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697483) roll requested 2024-12-07T00:51:37,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697491 2024-12-07T00:51:37,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697491 2024-12-07T00:51:37,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697491) roll requested 2024-12-07T00:51:37,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697501 2024-12-07T00:51:37,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697501 2024-12-07T00:51:37,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697501) roll requested 2024-12-07T00:51:37,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697511 2024-12-07T00:51:37,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697511 2024-12-07T00:51:37,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697511) roll requested 2024-12-07T00:51:37,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697518 2024-12-07T00:51:37,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697518 2024-12-07T00:51:37,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697518) roll requested 2024-12-07T00:51:37,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697527 2024-12-07T00:51:37,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697527 2024-12-07T00:51:37,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697527) roll requested 2024-12-07T00:51:37,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697534 2024-12-07T00:51:37,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697534 2024-12-07T00:51:37,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697534) roll requested 2024-12-07T00:51:37,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697542 2024-12-07T00:51:37,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697542 2024-12-07T00:51:37,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697542) roll requested 2024-12-07T00:51:37,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697551 2024-12-07T00:51:37,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697551 2024-12-07T00:51:37,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697551) roll requested 2024-12-07T00:51:37,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697562 2024-12-07T00:51:37,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697562 2024-12-07T00:51:37,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697562) roll requested 2024-12-07T00:51:37,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697568 2024-12-07T00:51:37,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697568 2024-12-07T00:51:37,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697568) roll requested 2024-12-07T00:51:37,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697574 2024-12-07T00:51:37,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697574 2024-12-07T00:51:37,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697574) roll requested 2024-12-07T00:51:37,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697582 2024-12-07T00:51:37,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697582 2024-12-07T00:51:37,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697582) roll requested 2024-12-07T00:51:37,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697590 2024-12-07T00:51:37,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697590 2024-12-07T00:51:37,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697590) roll requested 2024-12-07T00:51:37,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697599 2024-12-07T00:51:37,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697599 2024-12-07T00:51:37,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697599) roll requested 2024-12-07T00:51:37,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697608 2024-12-07T00:51:37,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697608 2024-12-07T00:51:37,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697608) roll requested 2024-12-07T00:51:37,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697616 2024-12-07T00:51:37,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697616 2024-12-07T00:51:37,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697616) roll requested 2024-12-07T00:51:37,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697624 2024-12-07T00:51:37,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697624 2024-12-07T00:51:37,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697624) roll requested 2024-12-07T00:51:37,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697634 2024-12-07T00:51:37,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697634 2024-12-07T00:51:37,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697634) roll requested 2024-12-07T00:51:37,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697644 2024-12-07T00:51:37,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697644 2024-12-07T00:51:37,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697644) roll requested 2024-12-07T00:51:37,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697652 2024-12-07T00:51:37,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697652 2024-12-07T00:51:37,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697652) roll requested 2024-12-07T00:51:37,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697658 2024-12-07T00:51:37,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697658 2024-12-07T00:51:37,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697658) roll requested 2024-12-07T00:51:37,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697663 2024-12-07T00:51:37,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697663 2024-12-07T00:51:37,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697663) roll requested 2024-12-07T00:51:37,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697669 2024-12-07T00:51:37,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697669 2024-12-07T00:51:37,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697669) roll requested 2024-12-07T00:51:37,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697676 2024-12-07T00:51:37,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697676 2024-12-07T00:51:37,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697676) roll requested 2024-12-07T00:51:37,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697684 2024-12-07T00:51:37,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697684 2024-12-07T00:51:37,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697684) roll requested 2024-12-07T00:51:37,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697692 2024-12-07T00:51:37,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697692 2024-12-07T00:51:37,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697692) roll requested 2024-12-07T00:51:37,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697701 2024-12-07T00:51:37,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697701 2024-12-07T00:51:37,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697701) roll requested 2024-12-07T00:51:37,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697709 2024-12-07T00:51:37,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697709 2024-12-07T00:51:37,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697709) roll requested 2024-12-07T00:51:37,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697717 2024-12-07T00:51:37,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697717 2024-12-07T00:51:37,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697717) roll requested 2024-12-07T00:51:37,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697724 2024-12-07T00:51:37,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697724 2024-12-07T00:51:37,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697724) roll requested 2024-12-07T00:51:37,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697731 2024-12-07T00:51:37,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697731 2024-12-07T00:51:37,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697731) roll requested 2024-12-07T00:51:37,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697738 2024-12-07T00:51:37,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697738 2024-12-07T00:51:37,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697738) roll requested 2024-12-07T00:51:37,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697746 2024-12-07T00:51:37,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697746 2024-12-07T00:51:37,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697746) roll requested 2024-12-07T00:51:37,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697755 2024-12-07T00:51:37,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697755 2024-12-07T00:51:37,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,762 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697755) roll requested 2024-12-07T00:51:37,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697763 2024-12-07T00:51:37,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697763 2024-12-07T00:51:37,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697763) roll requested 2024-12-07T00:51:37,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697770 2024-12-07T00:51:37,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697770 2024-12-07T00:51:37,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697770) roll requested 2024-12-07T00:51:37,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697777 2024-12-07T00:51:37,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697777 2024-12-07T00:51:37,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697777) roll requested 2024-12-07T00:51:37,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697786 2024-12-07T00:51:37,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697786 2024-12-07T00:51:37,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697786) roll requested 2024-12-07T00:51:37,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697795 2024-12-07T00:51:37,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697795 2024-12-07T00:51:37,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697795) roll requested 2024-12-07T00:51:37,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697805 2024-12-07T00:51:37,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697805 2024-12-07T00:51:37,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697805) roll requested 2024-12-07T00:51:37,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697814 2024-12-07T00:51:37,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697814 2024-12-07T00:51:37,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697814) roll requested 2024-12-07T00:51:37,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697820 2024-12-07T00:51:37,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697820 2024-12-07T00:51:37,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,826 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,826 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697820) roll requested 2024-12-07T00:51:37,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697826 2024-12-07T00:51:37,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697826 2024-12-07T00:51:37,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697826) roll requested 2024-12-07T00:51:37,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697833 2024-12-07T00:51:37,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697833 2024-12-07T00:51:37,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697833) roll requested 2024-12-07T00:51:37,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697840 2024-12-07T00:51:37,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697840 2024-12-07T00:51:37,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697840) roll requested 2024-12-07T00:51:37,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697849 2024-12-07T00:51:37,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697849 2024-12-07T00:51:37,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697849) roll requested 2024-12-07T00:51:37,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697856 2024-12-07T00:51:37,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697856 2024-12-07T00:51:37,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697856) roll requested 2024-12-07T00:51:37,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697864 2024-12-07T00:51:37,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697864 2024-12-07T00:51:37,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697864) roll requested 2024-12-07T00:51:37,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697871 2024-12-07T00:51:37,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697871 2024-12-07T00:51:37,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697871) roll requested 2024-12-07T00:51:37,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697876 2024-12-07T00:51:37,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697876 2024-12-07T00:51:37,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697876) roll requested 2024-12-07T00:51:37,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697882 2024-12-07T00:51:37,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697882 2024-12-07T00:51:37,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697882) roll requested 2024-12-07T00:51:37,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697889 2024-12-07T00:51:37,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697889 2024-12-07T00:51:37,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697889) roll requested 2024-12-07T00:51:37,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697897 2024-12-07T00:51:37,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697897 2024-12-07T00:51:37,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697897) roll requested 2024-12-07T00:51:37,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697905 2024-12-07T00:51:37,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697905 2024-12-07T00:51:37,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697905) roll requested 2024-12-07T00:51:37,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697912 2024-12-07T00:51:37,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697912 2024-12-07T00:51:37,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697912) roll requested 2024-12-07T00:51:37,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697920 2024-12-07T00:51:37,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697920 2024-12-07T00:51:37,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697920) roll requested 2024-12-07T00:51:37,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697931 2024-12-07T00:51:37,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697931 2024-12-07T00:51:37,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697931) roll requested 2024-12-07T00:51:37,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697937 2024-12-07T00:51:37,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697937 2024-12-07T00:51:37,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697937) roll requested 2024-12-07T00:51:37,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697943 2024-12-07T00:51:37,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697943 2024-12-07T00:51:37,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697943) roll requested 2024-12-07T00:51:37,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697949 2024-12-07T00:51:37,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697949 2024-12-07T00:51:37,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697949) roll requested 2024-12-07T00:51:37,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697956 2024-12-07T00:51:37,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697956 2024-12-07T00:51:37,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697956) roll requested 2024-12-07T00:51:37,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697964 2024-12-07T00:51:37,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697964 2024-12-07T00:51:37,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697964) roll requested 2024-12-07T00:51:37,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697972 2024-12-07T00:51:37,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697972 2024-12-07T00:51:37,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697972) roll requested 2024-12-07T00:51:37,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697981 2024-12-07T00:51:37,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697981 2024-12-07T00:51:37,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697981) roll requested 2024-12-07T00:51:37,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697990 2024-12-07T00:51:37,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:37,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697990 2024-12-07T00:51:37,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:37,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:37,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:37,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:37,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697990) roll requested 2024-12-07T00:51:37,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532697998 2024-12-07T00:51:38,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697998 2024-12-07T00:51:38,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532697998) roll requested 2024-12-07T00:51:38,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698006 2024-12-07T00:51:38,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532697998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698006 2024-12-07T00:51:38,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698006) roll requested 2024-12-07T00:51:38,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698013 2024-12-07T00:51:38,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698013 2024-12-07T00:51:38,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698013) roll requested 2024-12-07T00:51:38,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698019 2024-12-07T00:51:38,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698019 2024-12-07T00:51:38,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698019) roll requested 2024-12-07T00:51:38,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698026 2024-12-07T00:51:38,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698026 2024-12-07T00:51:38,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698026) roll requested 2024-12-07T00:51:38,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698033 2024-12-07T00:51:38,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698033 2024-12-07T00:51:38,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698033) roll requested 2024-12-07T00:51:38,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698040 2024-12-07T00:51:38,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698040 2024-12-07T00:51:38,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698040) roll requested 2024-12-07T00:51:38,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698048 2024-12-07T00:51:38,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698048 2024-12-07T00:51:38,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698048) roll requested 2024-12-07T00:51:38,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698057 2024-12-07T00:51:38,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698057 2024-12-07T00:51:38,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698057) roll requested 2024-12-07T00:51:38,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698064 2024-12-07T00:51:38,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698064 2024-12-07T00:51:38,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698064) roll requested 2024-12-07T00:51:38,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698071 2024-12-07T00:51:38,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698071 2024-12-07T00:51:38,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698071) roll requested 2024-12-07T00:51:38,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698077 2024-12-07T00:51:38,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698077 2024-12-07T00:51:38,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698077) roll requested 2024-12-07T00:51:38,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698084 2024-12-07T00:51:38,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698084 2024-12-07T00:51:38,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698084) roll requested 2024-12-07T00:51:38,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698093 2024-12-07T00:51:38,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698093 2024-12-07T00:51:38,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698093) roll requested 2024-12-07T00:51:38,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698101 2024-12-07T00:51:38,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698101 2024-12-07T00:51:38,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698101) roll requested 2024-12-07T00:51:38,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698109 2024-12-07T00:51:38,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698109 2024-12-07T00:51:38,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698109) roll requested 2024-12-07T00:51:38,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698117 2024-12-07T00:51:38,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698117 2024-12-07T00:51:38,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698117) roll requested 2024-12-07T00:51:38,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698125 2024-12-07T00:51:38,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698125 2024-12-07T00:51:38,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698125) roll requested 2024-12-07T00:51:38,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698135 2024-12-07T00:51:38,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698135 2024-12-07T00:51:38,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698135) roll requested 2024-12-07T00:51:38,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698143 2024-12-07T00:51:38,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698143 2024-12-07T00:51:38,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698143) roll requested 2024-12-07T00:51:38,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698151 2024-12-07T00:51:38,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698151 2024-12-07T00:51:38,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698151) roll requested 2024-12-07T00:51:38,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698157 2024-12-07T00:51:38,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698157 2024-12-07T00:51:38,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698157) roll requested 2024-12-07T00:51:38,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698162 2024-12-07T00:51:38,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698162 2024-12-07T00:51:38,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698162) roll requested 2024-12-07T00:51:38,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698168 2024-12-07T00:51:38,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698168 2024-12-07T00:51:38,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698168) roll requested 2024-12-07T00:51:38,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698174 2024-12-07T00:51:38,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698174 2024-12-07T00:51:38,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698174) roll requested 2024-12-07T00:51:38,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698182 2024-12-07T00:51:38,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698182 2024-12-07T00:51:38,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698182) roll requested 2024-12-07T00:51:38,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698189 2024-12-07T00:51:38,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698189 2024-12-07T00:51:38,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698189) roll requested 2024-12-07T00:51:38,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698197 2024-12-07T00:51:38,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698197 2024-12-07T00:51:38,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698197) roll requested 2024-12-07T00:51:38,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698204 2024-12-07T00:51:38,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698204 2024-12-07T00:51:38,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698204) roll requested 2024-12-07T00:51:38,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698215 2024-12-07T00:51:38,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698215 2024-12-07T00:51:38,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698215) roll requested 2024-12-07T00:51:38,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698224 2024-12-07T00:51:38,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698224 2024-12-07T00:51:38,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698224) roll requested 2024-12-07T00:51:38,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698229 2024-12-07T00:51:38,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698229 2024-12-07T00:51:38,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698229) roll requested 2024-12-07T00:51:38,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698236 2024-12-07T00:51:38,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698236 2024-12-07T00:51:38,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698236) roll requested 2024-12-07T00:51:38,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698243 2024-12-07T00:51:38,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698243 2024-12-07T00:51:38,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698243) roll requested 2024-12-07T00:51:38,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698251 2024-12-07T00:51:38,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698251 2024-12-07T00:51:38,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698251) roll requested 2024-12-07T00:51:38,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698260 2024-12-07T00:51:38,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698260 2024-12-07T00:51:38,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698260) roll requested 2024-12-07T00:51:38,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698269 2024-12-07T00:51:38,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698269 2024-12-07T00:51:38,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698269) roll requested 2024-12-07T00:51:38,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698276 2024-12-07T00:51:38,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698276 2024-12-07T00:51:38,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698276) roll requested 2024-12-07T00:51:38,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698285 2024-12-07T00:51:38,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698285 2024-12-07T00:51:38,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698285) roll requested 2024-12-07T00:51:38,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698294 2024-12-07T00:51:38,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698294 2024-12-07T00:51:38,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698294) roll requested 2024-12-07T00:51:38,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698301 2024-12-07T00:51:38,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698301 2024-12-07T00:51:38,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698301) roll requested 2024-12-07T00:51:38,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698308 2024-12-07T00:51:38,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698308 2024-12-07T00:51:38,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698308) roll requested 2024-12-07T00:51:38,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698316 2024-12-07T00:51:38,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698316 2024-12-07T00:51:38,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698316) roll requested 2024-12-07T00:51:38,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698374 2024-12-07T00:51:38,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698374 2024-12-07T00:51:38,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698374) roll requested 2024-12-07T00:51:38,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698380 2024-12-07T00:51:38,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698380 2024-12-07T00:51:38,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698380) roll requested 2024-12-07T00:51:38,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698386 2024-12-07T00:51:38,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698386 2024-12-07T00:51:38,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698386) roll requested 2024-12-07T00:51:38,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698391 2024-12-07T00:51:38,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698391 2024-12-07T00:51:38,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698391) roll requested 2024-12-07T00:51:38,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698397 2024-12-07T00:51:38,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698397 2024-12-07T00:51:38,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698397) roll requested 2024-12-07T00:51:38,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698402 2024-12-07T00:51:38,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698402 2024-12-07T00:51:38,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698402) roll requested 2024-12-07T00:51:38,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698408 2024-12-07T00:51:38,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698408 2024-12-07T00:51:38,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698408) roll requested 2024-12-07T00:51:38,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698414 2024-12-07T00:51:38,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698414 2024-12-07T00:51:38,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698414) roll requested 2024-12-07T00:51:38,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698420 2024-12-07T00:51:38,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698420 2024-12-07T00:51:38,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698420) roll requested 2024-12-07T00:51:38,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698426 2024-12-07T00:51:38,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698426 2024-12-07T00:51:38,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698426) roll requested 2024-12-07T00:51:38,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698431 2024-12-07T00:51:38,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698431 2024-12-07T00:51:38,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698431) roll requested 2024-12-07T00:51:38,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698437 2024-12-07T00:51:38,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698437 2024-12-07T00:51:38,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698437) roll requested 2024-12-07T00:51:38,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698442 2024-12-07T00:51:38,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698442 2024-12-07T00:51:38,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698442) roll requested 2024-12-07T00:51:38,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698448 2024-12-07T00:51:38,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698448 2024-12-07T00:51:38,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698448) roll requested 2024-12-07T00:51:38,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698455 2024-12-07T00:51:38,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698455 2024-12-07T00:51:38,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698455) roll requested 2024-12-07T00:51:38,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698462 2024-12-07T00:51:38,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698462 2024-12-07T00:51:38,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698462) roll requested 2024-12-07T00:51:38,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698468 2024-12-07T00:51:38,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698468 2024-12-07T00:51:38,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698468) roll requested 2024-12-07T00:51:38,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698473 2024-12-07T00:51:38,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698473 2024-12-07T00:51:38,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698473) roll requested 2024-12-07T00:51:38,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698479 2024-12-07T00:51:38,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698479 2024-12-07T00:51:38,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698479) roll requested 2024-12-07T00:51:38,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698484 2024-12-07T00:51:38,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698484 2024-12-07T00:51:38,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698484) roll requested 2024-12-07T00:51:38,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698490 2024-12-07T00:51:38,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698490 2024-12-07T00:51:38,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698490) roll requested 2024-12-07T00:51:38,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698495 2024-12-07T00:51:38,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698495 2024-12-07T00:51:38,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698495) roll requested 2024-12-07T00:51:38,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698500 2024-12-07T00:51:38,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698500 2024-12-07T00:51:38,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698500) roll requested 2024-12-07T00:51:38,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698505 2024-12-07T00:51:38,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698505 2024-12-07T00:51:38,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698505) roll requested 2024-12-07T00:51:38,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698512 2024-12-07T00:51:38,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698512 2024-12-07T00:51:38,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698512) roll requested 2024-12-07T00:51:38,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698518 2024-12-07T00:51:38,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698518 2024-12-07T00:51:38,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698518) roll requested 2024-12-07T00:51:38,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698524 2024-12-07T00:51:38,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698524 2024-12-07T00:51:38,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698524) roll requested 2024-12-07T00:51:38,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698529 2024-12-07T00:51:38,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698529 2024-12-07T00:51:38,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698529) roll requested 2024-12-07T00:51:38,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698535 2024-12-07T00:51:38,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698535 2024-12-07T00:51:38,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698535) roll requested 2024-12-07T00:51:38,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698542 2024-12-07T00:51:38,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698542 2024-12-07T00:51:38,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698542) roll requested 2024-12-07T00:51:38,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698548 2024-12-07T00:51:38,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698548 2024-12-07T00:51:38,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698548) roll requested 2024-12-07T00:51:38,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698555 2024-12-07T00:51:38,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698555 2024-12-07T00:51:38,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698555) roll requested 2024-12-07T00:51:38,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698561 2024-12-07T00:51:38,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698561 2024-12-07T00:51:38,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698561) roll requested 2024-12-07T00:51:38,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698567 2024-12-07T00:51:38,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698567 2024-12-07T00:51:38,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698567) roll requested 2024-12-07T00:51:38,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698572 2024-12-07T00:51:38,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698572 2024-12-07T00:51:38,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698572) roll requested 2024-12-07T00:51:38,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698578 2024-12-07T00:51:38,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698578 2024-12-07T00:51:38,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698578) roll requested 2024-12-07T00:51:38,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698583 2024-12-07T00:51:38,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698583 2024-12-07T00:51:38,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698583) roll requested 2024-12-07T00:51:38,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698589 2024-12-07T00:51:38,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698589 2024-12-07T00:51:38,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698589) roll requested 2024-12-07T00:51:38,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698595 2024-12-07T00:51:38,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698595 2024-12-07T00:51:38,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698595) roll requested 2024-12-07T00:51:38,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698601 2024-12-07T00:51:38,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698601 2024-12-07T00:51:38,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698601) roll requested 2024-12-07T00:51:38,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698608 2024-12-07T00:51:38,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698608 2024-12-07T00:51:38,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698608) roll requested 2024-12-07T00:51:38,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698614 2024-12-07T00:51:38,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698614 2024-12-07T00:51:38,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698614) roll requested 2024-12-07T00:51:38,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698620 2024-12-07T00:51:38,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698620 2024-12-07T00:51:38,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698620) roll requested 2024-12-07T00:51:38,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698630 2024-12-07T00:51:38,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698630 2024-12-07T00:51:38,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698630) roll requested 2024-12-07T00:51:38,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698640 2024-12-07T00:51:38,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698640 2024-12-07T00:51:38,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698640) roll requested 2024-12-07T00:51:38,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698647 2024-12-07T00:51:38,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698647 2024-12-07T00:51:38,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698647) roll requested 2024-12-07T00:51:38,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698654 2024-12-07T00:51:38,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698654 2024-12-07T00:51:38,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698654) roll requested 2024-12-07T00:51:38,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698660 2024-12-07T00:51:38,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698660 2024-12-07T00:51:38,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698660) roll requested 2024-12-07T00:51:38,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698666 2024-12-07T00:51:38,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698666 2024-12-07T00:51:38,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698666) roll requested 2024-12-07T00:51:38,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698671 2024-12-07T00:51:38,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698671 2024-12-07T00:51:38,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698671) roll requested 2024-12-07T00:51:38,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698677 2024-12-07T00:51:38,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698677 2024-12-07T00:51:38,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698677) roll requested 2024-12-07T00:51:38,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698683 2024-12-07T00:51:38,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698683 2024-12-07T00:51:38,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698683) roll requested 2024-12-07T00:51:38,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698689 2024-12-07T00:51:38,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698689 2024-12-07T00:51:38,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698689) roll requested 2024-12-07T00:51:38,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698695 2024-12-07T00:51:38,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698695 2024-12-07T00:51:38,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698695) roll requested 2024-12-07T00:51:38,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698701 2024-12-07T00:51:38,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698701 2024-12-07T00:51:38,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698701) roll requested 2024-12-07T00:51:38,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698707 2024-12-07T00:51:38,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698707 2024-12-07T00:51:38,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698707) roll requested 2024-12-07T00:51:38,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698714 2024-12-07T00:51:38,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698714 2024-12-07T00:51:38,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698714) roll requested 2024-12-07T00:51:38,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698720 2024-12-07T00:51:38,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698720 2024-12-07T00:51:38,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698720) roll requested 2024-12-07T00:51:38,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698726 2024-12-07T00:51:38,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698726 2024-12-07T00:51:38,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698726) roll requested 2024-12-07T00:51:38,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698732 2024-12-07T00:51:38,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698732 2024-12-07T00:51:38,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698732) roll requested 2024-12-07T00:51:38,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698738 2024-12-07T00:51:38,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698738 2024-12-07T00:51:38,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698738) roll requested 2024-12-07T00:51:38,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698744 2024-12-07T00:51:38,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698744 2024-12-07T00:51:38,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698744) roll requested 2024-12-07T00:51:38,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698750 2024-12-07T00:51:38,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698750 2024-12-07T00:51:38,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698750) roll requested 2024-12-07T00:51:38,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698756 2024-12-07T00:51:38,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698756 2024-12-07T00:51:38,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698756) roll requested 2024-12-07T00:51:38,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698761 2024-12-07T00:51:38,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698761 2024-12-07T00:51:38,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698761) roll requested 2024-12-07T00:51:38,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698767 2024-12-07T00:51:38,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698767 2024-12-07T00:51:38,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698767) roll requested 2024-12-07T00:51:38,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698773 2024-12-07T00:51:38,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698773 2024-12-07T00:51:38,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698773) roll requested 2024-12-07T00:51:38,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698779 2024-12-07T00:51:38,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698779 2024-12-07T00:51:38,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698779) roll requested 2024-12-07T00:51:38,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698785 2024-12-07T00:51:38,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698785 2024-12-07T00:51:38,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698785) roll requested 2024-12-07T00:51:38,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698791 2024-12-07T00:51:38,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698791 2024-12-07T00:51:38,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698791) roll requested 2024-12-07T00:51:38,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698797 2024-12-07T00:51:38,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698797 2024-12-07T00:51:38,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698797) roll requested 2024-12-07T00:51:38,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698803 2024-12-07T00:51:38,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698803 2024-12-07T00:51:38,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698803) roll requested 2024-12-07T00:51:38,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698809 2024-12-07T00:51:38,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698809 2024-12-07T00:51:38,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698809) roll requested 2024-12-07T00:51:38,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698815 2024-12-07T00:51:38,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698815 2024-12-07T00:51:38,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698815) roll requested 2024-12-07T00:51:38,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698821 2024-12-07T00:51:38,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698821 2024-12-07T00:51:38,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698821) roll requested 2024-12-07T00:51:38,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698828 2024-12-07T00:51:38,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698828 2024-12-07T00:51:38,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698828) roll requested 2024-12-07T00:51:38,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698834 2024-12-07T00:51:38,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698834 2024-12-07T00:51:38,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698834) roll requested 2024-12-07T00:51:38,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698839 2024-12-07T00:51:38,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698839 2024-12-07T00:51:38,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698839) roll requested 2024-12-07T00:51:38,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698845 2024-12-07T00:51:38,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698845 2024-12-07T00:51:38,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698845) roll requested 2024-12-07T00:51:38,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698852 2024-12-07T00:51:38,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698852 2024-12-07T00:51:38,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698852) roll requested 2024-12-07T00:51:38,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698858 2024-12-07T00:51:38,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698858 2024-12-07T00:51:38,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698858) roll requested 2024-12-07T00:51:38,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698864 2024-12-07T00:51:38,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698864 2024-12-07T00:51:38,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698864) roll requested 2024-12-07T00:51:38,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698870 2024-12-07T00:51:38,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698870 2024-12-07T00:51:38,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698870) roll requested 2024-12-07T00:51:38,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698876 2024-12-07T00:51:38,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698876 2024-12-07T00:51:38,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698876) roll requested 2024-12-07T00:51:38,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698883 2024-12-07T00:51:38,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698883 2024-12-07T00:51:38,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698883) roll requested 2024-12-07T00:51:38,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698889 2024-12-07T00:51:38,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698889 2024-12-07T00:51:38,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698889) roll requested 2024-12-07T00:51:38,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698896 2024-12-07T00:51:38,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698896 2024-12-07T00:51:38,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698896) roll requested 2024-12-07T00:51:38,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698902 2024-12-07T00:51:38,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698902 2024-12-07T00:51:38,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698902) roll requested 2024-12-07T00:51:38,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698909 2024-12-07T00:51:38,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698909 2024-12-07T00:51:38,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698909) roll requested 2024-12-07T00:51:38,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698918 2024-12-07T00:51:38,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698918 2024-12-07T00:51:38,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698918) roll requested 2024-12-07T00:51:38,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698928 2024-12-07T00:51:38,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698928 2024-12-07T00:51:38,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698928) roll requested 2024-12-07T00:51:38,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698934 2024-12-07T00:51:38,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698934 2024-12-07T00:51:38,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698934) roll requested 2024-12-07T00:51:38,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698941 2024-12-07T00:51:38,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698941 2024-12-07T00:51:38,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698941) roll requested 2024-12-07T00:51:38,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698947 2024-12-07T00:51:38,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698947 2024-12-07T00:51:38,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698947) roll requested 2024-12-07T00:51:38,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698953 2024-12-07T00:51:38,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698953 2024-12-07T00:51:38,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698953) roll requested 2024-12-07T00:51:38,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698958 2024-12-07T00:51:38,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698958 2024-12-07T00:51:38,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698958) roll requested 2024-12-07T00:51:38,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698964 2024-12-07T00:51:38,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698964 2024-12-07T00:51:38,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698964) roll requested 2024-12-07T00:51:38,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698970 2024-12-07T00:51:38,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698970 2024-12-07T00:51:38,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698970) roll requested 2024-12-07T00:51:38,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698976 2024-12-07T00:51:38,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698976 2024-12-07T00:51:38,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698976) roll requested 2024-12-07T00:51:38,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698982 2024-12-07T00:51:38,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698982 2024-12-07T00:51:38,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698982) roll requested 2024-12-07T00:51:38,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698987 2024-12-07T00:51:38,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698987 2024-12-07T00:51:38,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:38,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:38,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:38,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:38,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698987) roll requested 2024-12-07T00:51:38,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532698993 2024-12-07T00:51:38,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:38,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698993 2024-12-07T00:51:38,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532698993) roll requested 2024-12-07T00:51:39,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699000 2024-12-07T00:51:39,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532698993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699000 2024-12-07T00:51:39,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699000) roll requested 2024-12-07T00:51:39,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699006 2024-12-07T00:51:39,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699006 2024-12-07T00:51:39,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699006) roll requested 2024-12-07T00:51:39,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699011 2024-12-07T00:51:39,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699011 2024-12-07T00:51:39,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699011) roll requested 2024-12-07T00:51:39,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699017 2024-12-07T00:51:39,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699017 2024-12-07T00:51:39,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699017) roll requested 2024-12-07T00:51:39,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699023 2024-12-07T00:51:39,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699023 2024-12-07T00:51:39,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699023) roll requested 2024-12-07T00:51:39,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699029 2024-12-07T00:51:39,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699029 2024-12-07T00:51:39,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699029) roll requested 2024-12-07T00:51:39,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699036 2024-12-07T00:51:39,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699036 2024-12-07T00:51:39,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699036) roll requested 2024-12-07T00:51:39,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699042 2024-12-07T00:51:39,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699042 2024-12-07T00:51:39,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699042) roll requested 2024-12-07T00:51:39,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699048 2024-12-07T00:51:39,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699048 2024-12-07T00:51:39,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699048) roll requested 2024-12-07T00:51:39,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699054 2024-12-07T00:51:39,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699054 2024-12-07T00:51:39,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699054) roll requested 2024-12-07T00:51:39,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699060 2024-12-07T00:51:39,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699060 2024-12-07T00:51:39,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699060) roll requested 2024-12-07T00:51:39,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699066 2024-12-07T00:51:39,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699066 2024-12-07T00:51:39,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699066) roll requested 2024-12-07T00:51:39,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699071 2024-12-07T00:51:39,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699071 2024-12-07T00:51:39,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699071) roll requested 2024-12-07T00:51:39,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699077 2024-12-07T00:51:39,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699077 2024-12-07T00:51:39,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699077) roll requested 2024-12-07T00:51:39,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699083 2024-12-07T00:51:39,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699083 2024-12-07T00:51:39,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699083) roll requested 2024-12-07T00:51:39,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699089 2024-12-07T00:51:39,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699089 2024-12-07T00:51:39,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699089) roll requested 2024-12-07T00:51:39,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699095 2024-12-07T00:51:39,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699095 2024-12-07T00:51:39,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699095) roll requested 2024-12-07T00:51:39,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699100 2024-12-07T00:51:39,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699100 2024-12-07T00:51:39,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699100) roll requested 2024-12-07T00:51:39,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699106 2024-12-07T00:51:39,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699106 2024-12-07T00:51:39,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699106) roll requested 2024-12-07T00:51:39,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699113 2024-12-07T00:51:39,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699113 2024-12-07T00:51:39,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699113) roll requested 2024-12-07T00:51:39,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699119 2024-12-07T00:51:39,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699119 2024-12-07T00:51:39,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699119) roll requested 2024-12-07T00:51:39,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699125 2024-12-07T00:51:39,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699125 2024-12-07T00:51:39,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699125) roll requested 2024-12-07T00:51:39,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699130 2024-12-07T00:51:39,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699130 2024-12-07T00:51:39,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699130) roll requested 2024-12-07T00:51:39,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699136 2024-12-07T00:51:39,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699136 2024-12-07T00:51:39,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699136) roll requested 2024-12-07T00:51:39,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699142 2024-12-07T00:51:39,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699142 2024-12-07T00:51:39,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699142) roll requested 2024-12-07T00:51:39,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699149 2024-12-07T00:51:39,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699149 2024-12-07T00:51:39,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699149) roll requested 2024-12-07T00:51:39,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699156 2024-12-07T00:51:39,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699156 2024-12-07T00:51:39,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699156) roll requested 2024-12-07T00:51:39,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699162 2024-12-07T00:51:39,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699162 2024-12-07T00:51:39,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699162) roll requested 2024-12-07T00:51:39,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699167 2024-12-07T00:51:39,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699167 2024-12-07T00:51:39,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699167) roll requested 2024-12-07T00:51:39,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699173 2024-12-07T00:51:39,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699173 2024-12-07T00:51:39,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699173) roll requested 2024-12-07T00:51:39,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699180 2024-12-07T00:51:39,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699180 2024-12-07T00:51:39,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699180) roll requested 2024-12-07T00:51:39,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699186 2024-12-07T00:51:39,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699186 2024-12-07T00:51:39,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699186) roll requested 2024-12-07T00:51:39,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699192 2024-12-07T00:51:39,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699192 2024-12-07T00:51:39,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699192) roll requested 2024-12-07T00:51:39,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699198 2024-12-07T00:51:39,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699198 2024-12-07T00:51:39,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699198) roll requested 2024-12-07T00:51:39,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699203 2024-12-07T00:51:39,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699203 2024-12-07T00:51:39,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699203) roll requested 2024-12-07T00:51:39,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699209 2024-12-07T00:51:39,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699209 2024-12-07T00:51:39,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699209) roll requested 2024-12-07T00:51:39,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699215 2024-12-07T00:51:39,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699215 2024-12-07T00:51:39,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699215) roll requested 2024-12-07T00:51:39,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699221 2024-12-07T00:51:39,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699221 2024-12-07T00:51:39,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699221) roll requested 2024-12-07T00:51:39,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699228 2024-12-07T00:51:39,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699228 2024-12-07T00:51:39,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699228) roll requested 2024-12-07T00:51:39,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699233 2024-12-07T00:51:39,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699233 2024-12-07T00:51:39,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699233) roll requested 2024-12-07T00:51:39,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699239 2024-12-07T00:51:39,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699239 2024-12-07T00:51:39,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699239) roll requested 2024-12-07T00:51:39,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699245 2024-12-07T00:51:39,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699245 2024-12-07T00:51:39,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699245) roll requested 2024-12-07T00:51:39,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699250 2024-12-07T00:51:39,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699250 2024-12-07T00:51:39,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699250) roll requested 2024-12-07T00:51:39,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699256 2024-12-07T00:51:39,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699256 2024-12-07T00:51:39,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699256) roll requested 2024-12-07T00:51:39,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699265 2024-12-07T00:51:39,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699265 2024-12-07T00:51:39,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699265) roll requested 2024-12-07T00:51:39,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699275 2024-12-07T00:51:39,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699275 2024-12-07T00:51:39,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699275) roll requested 2024-12-07T00:51:39,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699281 2024-12-07T00:51:39,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699281 2024-12-07T00:51:39,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699281) roll requested 2024-12-07T00:51:39,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699289 2024-12-07T00:51:39,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699289 2024-12-07T00:51:39,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699289) roll requested 2024-12-07T00:51:39,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699294 2024-12-07T00:51:39,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699294 2024-12-07T00:51:39,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699294) roll requested 2024-12-07T00:51:39,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699300 2024-12-07T00:51:39,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699300 2024-12-07T00:51:39,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699300) roll requested 2024-12-07T00:51:39,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699306 2024-12-07T00:51:39,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699306 2024-12-07T00:51:39,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699306) roll requested 2024-12-07T00:51:39,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699312 2024-12-07T00:51:39,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699312 2024-12-07T00:51:39,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699312) roll requested 2024-12-07T00:51:39,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699318 2024-12-07T00:51:39,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699318 2024-12-07T00:51:39,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699318) roll requested 2024-12-07T00:51:39,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699327 2024-12-07T00:51:39,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699327 2024-12-07T00:51:39,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699327) roll requested 2024-12-07T00:51:39,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699333 2024-12-07T00:51:39,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699333 2024-12-07T00:51:39,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699333) roll requested 2024-12-07T00:51:39,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699340 2024-12-07T00:51:39,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699340 2024-12-07T00:51:39,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699340) roll requested 2024-12-07T00:51:39,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699346 2024-12-07T00:51:39,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699346 2024-12-07T00:51:39,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699346) roll requested 2024-12-07T00:51:39,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699352 2024-12-07T00:51:39,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699352 2024-12-07T00:51:39,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699352) roll requested 2024-12-07T00:51:39,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699358 2024-12-07T00:51:39,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699358 2024-12-07T00:51:39,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699358) roll requested 2024-12-07T00:51:39,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699364 2024-12-07T00:51:39,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699364 2024-12-07T00:51:39,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699364) roll requested 2024-12-07T00:51:39,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699370 2024-12-07T00:51:39,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699370 2024-12-07T00:51:39,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699370) roll requested 2024-12-07T00:51:39,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699375 2024-12-07T00:51:39,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699375 2024-12-07T00:51:39,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699375) roll requested 2024-12-07T00:51:39,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699382 2024-12-07T00:51:39,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699382 2024-12-07T00:51:39,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699382) roll requested 2024-12-07T00:51:39,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699389 2024-12-07T00:51:39,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699389 2024-12-07T00:51:39,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699389) roll requested 2024-12-07T00:51:39,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699395 2024-12-07T00:51:39,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699395 2024-12-07T00:51:39,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699395) roll requested 2024-12-07T00:51:39,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699401 2024-12-07T00:51:39,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699401 2024-12-07T00:51:39,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699401) roll requested 2024-12-07T00:51:39,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699406 2024-12-07T00:51:39,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699406 2024-12-07T00:51:39,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699406) roll requested 2024-12-07T00:51:39,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699412 2024-12-07T00:51:39,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699412 2024-12-07T00:51:39,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699412) roll requested 2024-12-07T00:51:39,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699417 2024-12-07T00:51:39,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699417 2024-12-07T00:51:39,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699417) roll requested 2024-12-07T00:51:39,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699423 2024-12-07T00:51:39,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699423 2024-12-07T00:51:39,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699423) roll requested 2024-12-07T00:51:39,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699429 2024-12-07T00:51:39,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699429 2024-12-07T00:51:39,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699429) roll requested 2024-12-07T00:51:39,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699435 2024-12-07T00:51:39,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699435 2024-12-07T00:51:39,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699435) roll requested 2024-12-07T00:51:39,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699441 2024-12-07T00:51:39,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699441 2024-12-07T00:51:39,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699441) roll requested 2024-12-07T00:51:39,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699447 2024-12-07T00:51:39,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699447 2024-12-07T00:51:39,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699447) roll requested 2024-12-07T00:51:39,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699452 2024-12-07T00:51:39,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699452 2024-12-07T00:51:39,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699452) roll requested 2024-12-07T00:51:39,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699458 2024-12-07T00:51:39,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699458 2024-12-07T00:51:39,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699458) roll requested 2024-12-07T00:51:39,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699464 2024-12-07T00:51:39,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699464 2024-12-07T00:51:39,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699464) roll requested 2024-12-07T00:51:39,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699470 2024-12-07T00:51:39,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699470 2024-12-07T00:51:39,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699470) roll requested 2024-12-07T00:51:39,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699476 2024-12-07T00:51:39,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699476 2024-12-07T00:51:39,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699476) roll requested 2024-12-07T00:51:39,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699482 2024-12-07T00:51:39,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699482 2024-12-07T00:51:39,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699482) roll requested 2024-12-07T00:51:39,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699488 2024-12-07T00:51:39,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699488 2024-12-07T00:51:39,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699488) roll requested 2024-12-07T00:51:39,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699494 2024-12-07T00:51:39,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699494 2024-12-07T00:51:39,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699494) roll requested 2024-12-07T00:51:39,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699500 2024-12-07T00:51:39,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699500 2024-12-07T00:51:39,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699500) roll requested 2024-12-07T00:51:39,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699506 2024-12-07T00:51:39,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699506 2024-12-07T00:51:39,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699506) roll requested 2024-12-07T00:51:39,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699512 2024-12-07T00:51:39,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699512 2024-12-07T00:51:39,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699512) roll requested 2024-12-07T00:51:39,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699519 2024-12-07T00:51:39,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699519 2024-12-07T00:51:39,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699519) roll requested 2024-12-07T00:51:39,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699524 2024-12-07T00:51:39,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699524 2024-12-07T00:51:39,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699524) roll requested 2024-12-07T00:51:39,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699530 2024-12-07T00:51:39,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699530 2024-12-07T00:51:39,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699530) roll requested 2024-12-07T00:51:39,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699536 2024-12-07T00:51:39,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699536 2024-12-07T00:51:39,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699536) roll requested 2024-12-07T00:51:39,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699542 2024-12-07T00:51:39,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699542 2024-12-07T00:51:39,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699542) roll requested 2024-12-07T00:51:39,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699548 2024-12-07T00:51:39,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699548 2024-12-07T00:51:39,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699548) roll requested 2024-12-07T00:51:39,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699554 2024-12-07T00:51:39,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699554 2024-12-07T00:51:39,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699554) roll requested 2024-12-07T00:51:39,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699560 2024-12-07T00:51:39,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699560 2024-12-07T00:51:39,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699560) roll requested 2024-12-07T00:51:39,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699567 2024-12-07T00:51:39,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699567 2024-12-07T00:51:39,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699567) roll requested 2024-12-07T00:51:39,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699573 2024-12-07T00:51:39,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699573 2024-12-07T00:51:39,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699573) roll requested 2024-12-07T00:51:39,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699579 2024-12-07T00:51:39,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699579 2024-12-07T00:51:39,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699579) roll requested 2024-12-07T00:51:39,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699585 2024-12-07T00:51:39,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699585 2024-12-07T00:51:39,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699585) roll requested 2024-12-07T00:51:39,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699591 2024-12-07T00:51:39,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699591 2024-12-07T00:51:39,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699591) roll requested 2024-12-07T00:51:39,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699596 2024-12-07T00:51:39,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699596 2024-12-07T00:51:39,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699596) roll requested 2024-12-07T00:51:39,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699602 2024-12-07T00:51:39,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699602 2024-12-07T00:51:39,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699602) roll requested 2024-12-07T00:51:39,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699608 2024-12-07T00:51:39,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699608 2024-12-07T00:51:39,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699608) roll requested 2024-12-07T00:51:39,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699618 2024-12-07T00:51:39,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699618 2024-12-07T00:51:39,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699618) roll requested 2024-12-07T00:51:39,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699626 2024-12-07T00:51:39,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699626 2024-12-07T00:51:39,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699626) roll requested 2024-12-07T00:51:39,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699632 2024-12-07T00:51:39,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699632 2024-12-07T00:51:39,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699632) roll requested 2024-12-07T00:51:39,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699639 2024-12-07T00:51:39,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699639 2024-12-07T00:51:39,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699639) roll requested 2024-12-07T00:51:39,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699645 2024-12-07T00:51:39,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699645 2024-12-07T00:51:39,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699645) roll requested 2024-12-07T00:51:39,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699650 2024-12-07T00:51:39,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699650 2024-12-07T00:51:39,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699650) roll requested 2024-12-07T00:51:39,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699656 2024-12-07T00:51:39,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699656 2024-12-07T00:51:39,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699656) roll requested 2024-12-07T00:51:39,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699662 2024-12-07T00:51:39,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699662 2024-12-07T00:51:39,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699662) roll requested 2024-12-07T00:51:39,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699667 2024-12-07T00:51:39,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699667 2024-12-07T00:51:39,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699667) roll requested 2024-12-07T00:51:39,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699672 2024-12-07T00:51:39,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699672 2024-12-07T00:51:39,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699672) roll requested 2024-12-07T00:51:39,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699679 2024-12-07T00:51:39,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699679 2024-12-07T00:51:39,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699679) roll requested 2024-12-07T00:51:39,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699684 2024-12-07T00:51:39,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699684 2024-12-07T00:51:39,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699684) roll requested 2024-12-07T00:51:39,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699690 2024-12-07T00:51:39,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699690 2024-12-07T00:51:39,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699690) roll requested 2024-12-07T00:51:39,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699696 2024-12-07T00:51:39,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699696 2024-12-07T00:51:39,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699696) roll requested 2024-12-07T00:51:39,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699701 2024-12-07T00:51:39,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699701 2024-12-07T00:51:39,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699701) roll requested 2024-12-07T00:51:39,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699707 2024-12-07T00:51:39,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699707 2024-12-07T00:51:39,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699707) roll requested 2024-12-07T00:51:39,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699712 2024-12-07T00:51:39,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699712 2024-12-07T00:51:39,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699712) roll requested 2024-12-07T00:51:39,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699718 2024-12-07T00:51:39,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699718 2024-12-07T00:51:39,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699718) roll requested 2024-12-07T00:51:39,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699724 2024-12-07T00:51:39,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699724 2024-12-07T00:51:39,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699724) roll requested 2024-12-07T00:51:39,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699730 2024-12-07T00:51:39,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699730 2024-12-07T00:51:39,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699730) roll requested 2024-12-07T00:51:39,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699737 2024-12-07T00:51:39,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699737 2024-12-07T00:51:39,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699737) roll requested 2024-12-07T00:51:39,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699743 2024-12-07T00:51:39,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699743 2024-12-07T00:51:39,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699743) roll requested 2024-12-07T00:51:39,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699749 2024-12-07T00:51:39,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699749 2024-12-07T00:51:39,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699749) roll requested 2024-12-07T00:51:39,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699756 2024-12-07T00:51:39,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699756 2024-12-07T00:51:39,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699756) roll requested 2024-12-07T00:51:39,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699763 2024-12-07T00:51:39,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699763 2024-12-07T00:51:39,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699763) roll requested 2024-12-07T00:51:39,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699769 2024-12-07T00:51:39,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699769 2024-12-07T00:51:39,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699769) roll requested 2024-12-07T00:51:39,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699774 2024-12-07T00:51:39,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699774 2024-12-07T00:51:39,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699774) roll requested 2024-12-07T00:51:39,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699782 2024-12-07T00:51:39,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699782 2024-12-07T00:51:39,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699782) roll requested 2024-12-07T00:51:39,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699788 2024-12-07T00:51:39,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699788 2024-12-07T00:51:39,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699788) roll requested 2024-12-07T00:51:39,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699794 2024-12-07T00:51:39,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699794 2024-12-07T00:51:39,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699794) roll requested 2024-12-07T00:51:39,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699800 2024-12-07T00:51:39,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699800 2024-12-07T00:51:39,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699800) roll requested 2024-12-07T00:51:39,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699806 2024-12-07T00:51:39,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699806 2024-12-07T00:51:39,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699806) roll requested 2024-12-07T00:51:39,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699812 2024-12-07T00:51:39,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699812 2024-12-07T00:51:39,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699812) roll requested 2024-12-07T00:51:39,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699818 2024-12-07T00:51:39,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699818 2024-12-07T00:51:39,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699818) roll requested 2024-12-07T00:51:39,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699825 2024-12-07T00:51:39,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699825 2024-12-07T00:51:39,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699825) roll requested 2024-12-07T00:51:39,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699830 2024-12-07T00:51:39,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699830 2024-12-07T00:51:39,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699830) roll requested 2024-12-07T00:51:39,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699836 2024-12-07T00:51:39,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699836 2024-12-07T00:51:39,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699836) roll requested 2024-12-07T00:51:39,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699842 2024-12-07T00:51:39,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699842 2024-12-07T00:51:39,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699842) roll requested 2024-12-07T00:51:39,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699848 2024-12-07T00:51:39,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699848 2024-12-07T00:51:39,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699848) roll requested 2024-12-07T00:51:39,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699853 2024-12-07T00:51:39,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699853 2024-12-07T00:51:39,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699853) roll requested 2024-12-07T00:51:39,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699860 2024-12-07T00:51:39,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699860 2024-12-07T00:51:39,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699860) roll requested 2024-12-07T00:51:39,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699868 2024-12-07T00:51:39,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699868 2024-12-07T00:51:39,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699868) roll requested 2024-12-07T00:51:39,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699876 2024-12-07T00:51:39,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699876 2024-12-07T00:51:39,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699876) roll requested 2024-12-07T00:51:39,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699881 2024-12-07T00:51:39,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699881 2024-12-07T00:51:39,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699881) roll requested 2024-12-07T00:51:39,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699887 2024-12-07T00:51:39,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699887 2024-12-07T00:51:39,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699887) roll requested 2024-12-07T00:51:39,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699894 2024-12-07T00:51:39,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699894 2024-12-07T00:51:39,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699894) roll requested 2024-12-07T00:51:39,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699900 2024-12-07T00:51:39,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699900 2024-12-07T00:51:39,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699900) roll requested 2024-12-07T00:51:39,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699906 2024-12-07T00:51:39,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699906 2024-12-07T00:51:39,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699906) roll requested 2024-12-07T00:51:39,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699917 2024-12-07T00:51:39,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699917 2024-12-07T00:51:39,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699917) roll requested 2024-12-07T00:51:39,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699927 2024-12-07T00:51:39,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699927 2024-12-07T00:51:39,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699927) roll requested 2024-12-07T00:51:39,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699934 2024-12-07T00:51:39,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699934 2024-12-07T00:51:39,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699934) roll requested 2024-12-07T00:51:39,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699940 2024-12-07T00:51:39,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699940 2024-12-07T00:51:39,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699940) roll requested 2024-12-07T00:51:39,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699947 2024-12-07T00:51:39,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699947 2024-12-07T00:51:39,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699947) roll requested 2024-12-07T00:51:39,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699956 2024-12-07T00:51:39,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699956 2024-12-07T00:51:39,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699956) roll requested 2024-12-07T00:51:39,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699964 2024-12-07T00:51:39,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699964 2024-12-07T00:51:39,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699964) roll requested 2024-12-07T00:51:39,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699971 2024-12-07T00:51:39,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699971 2024-12-07T00:51:39,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699971) roll requested 2024-12-07T00:51:39,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699977 2024-12-07T00:51:39,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699977 2024-12-07T00:51:39,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699977) roll requested 2024-12-07T00:51:39,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699984 2024-12-07T00:51:39,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699984 2024-12-07T00:51:39,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699984) roll requested 2024-12-07T00:51:39,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699989 2024-12-07T00:51:39,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:39,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699989 2024-12-07T00:51:39,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:39,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:39,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:39,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:39,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699989) roll requested 2024-12-07T00:51:39,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532699996 2024-12-07T00:51:40,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699996 2024-12-07T00:51:40,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532699996) roll requested 2024-12-07T00:51:40,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700002 2024-12-07T00:51:40,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532699996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700002 2024-12-07T00:51:40,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700002) roll requested 2024-12-07T00:51:40,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700008 2024-12-07T00:51:40,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700008 2024-12-07T00:51:40,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700008) roll requested 2024-12-07T00:51:40,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700013 2024-12-07T00:51:40,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700013 2024-12-07T00:51:40,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700013) roll requested 2024-12-07T00:51:40,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700019 2024-12-07T00:51:40,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700019 2024-12-07T00:51:40,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700019) roll requested 2024-12-07T00:51:40,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700025 2024-12-07T00:51:40,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700025 2024-12-07T00:51:40,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700025) roll requested 2024-12-07T00:51:40,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700031 2024-12-07T00:51:40,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700031 2024-12-07T00:51:40,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700031) roll requested 2024-12-07T00:51:40,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700037 2024-12-07T00:51:40,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700037 2024-12-07T00:51:40,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700037) roll requested 2024-12-07T00:51:40,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700043 2024-12-07T00:51:40,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700043 2024-12-07T00:51:40,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700043) roll requested 2024-12-07T00:51:40,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700049 2024-12-07T00:51:40,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700049 2024-12-07T00:51:40,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700049) roll requested 2024-12-07T00:51:40,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700055 2024-12-07T00:51:40,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700055 2024-12-07T00:51:40,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700055) roll requested 2024-12-07T00:51:40,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700061 2024-12-07T00:51:40,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700061 2024-12-07T00:51:40,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700061) roll requested 2024-12-07T00:51:40,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700068 2024-12-07T00:51:40,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700068 2024-12-07T00:51:40,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700068) roll requested 2024-12-07T00:51:40,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700074 2024-12-07T00:51:40,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700074 2024-12-07T00:51:40,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700074) roll requested 2024-12-07T00:51:40,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700079 2024-12-07T00:51:40,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700079 2024-12-07T00:51:40,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700079) roll requested 2024-12-07T00:51:40,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700084 2024-12-07T00:51:40,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700084 2024-12-07T00:51:40,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700084) roll requested 2024-12-07T00:51:40,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700090 2024-12-07T00:51:40,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700090 2024-12-07T00:51:40,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700090) roll requested 2024-12-07T00:51:40,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700095 2024-12-07T00:51:40,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700095 2024-12-07T00:51:40,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700095) roll requested 2024-12-07T00:51:40,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700100 2024-12-07T00:51:40,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700100 2024-12-07T00:51:40,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700100) roll requested 2024-12-07T00:51:40,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700105 2024-12-07T00:51:40,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700105 2024-12-07T00:51:40,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700105) roll requested 2024-12-07T00:51:40,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700111 2024-12-07T00:51:40,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700111 2024-12-07T00:51:40,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700111) roll requested 2024-12-07T00:51:40,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700117 2024-12-07T00:51:40,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700117 2024-12-07T00:51:40,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700117) roll requested 2024-12-07T00:51:40,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700124 2024-12-07T00:51:40,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700124 2024-12-07T00:51:40,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700124) roll requested 2024-12-07T00:51:40,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700130 2024-12-07T00:51:40,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700130 2024-12-07T00:51:40,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700130) roll requested 2024-12-07T00:51:40,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700137 2024-12-07T00:51:40,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700137 2024-12-07T00:51:40,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700137) roll requested 2024-12-07T00:51:40,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700143 2024-12-07T00:51:40,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700143 2024-12-07T00:51:40,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700143) roll requested 2024-12-07T00:51:40,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700151 2024-12-07T00:51:40,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700151 2024-12-07T00:51:40,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700151) roll requested 2024-12-07T00:51:40,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700156 2024-12-07T00:51:40,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700156 2024-12-07T00:51:40,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700156) roll requested 2024-12-07T00:51:40,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700162 2024-12-07T00:51:40,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700162 2024-12-07T00:51:40,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700162) roll requested 2024-12-07T00:51:40,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700168 2024-12-07T00:51:40,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700168 2024-12-07T00:51:40,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700168) roll requested 2024-12-07T00:51:40,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700174 2024-12-07T00:51:40,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700174 2024-12-07T00:51:40,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700174) roll requested 2024-12-07T00:51:40,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700180 2024-12-07T00:51:40,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700180 2024-12-07T00:51:40,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700180) roll requested 2024-12-07T00:51:40,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700185 2024-12-07T00:51:40,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700185 2024-12-07T00:51:40,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700185) roll requested 2024-12-07T00:51:40,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700190 2024-12-07T00:51:40,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700190 2024-12-07T00:51:40,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700190) roll requested 2024-12-07T00:51:40,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700195 2024-12-07T00:51:40,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700195 2024-12-07T00:51:40,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700195) roll requested 2024-12-07T00:51:40,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700202 2024-12-07T00:51:40,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700202 2024-12-07T00:51:40,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700202) roll requested 2024-12-07T00:51:40,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700210 2024-12-07T00:51:40,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700210 2024-12-07T00:51:40,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700210) roll requested 2024-12-07T00:51:40,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700218 2024-12-07T00:51:40,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700218 2024-12-07T00:51:40,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700218) roll requested 2024-12-07T00:51:40,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700227 2024-12-07T00:51:40,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700227 2024-12-07T00:51:40,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700227) roll requested 2024-12-07T00:51:40,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700237 2024-12-07T00:51:40,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700237 2024-12-07T00:51:40,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700237) roll requested 2024-12-07T00:51:40,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700248 2024-12-07T00:51:40,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700248 2024-12-07T00:51:40,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700248) roll requested 2024-12-07T00:51:40,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700257 2024-12-07T00:51:40,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700257 2024-12-07T00:51:40,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700257) roll requested 2024-12-07T00:51:40,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700264 2024-12-07T00:51:40,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700264 2024-12-07T00:51:40,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700264) roll requested 2024-12-07T00:51:40,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700270 2024-12-07T00:51:40,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700270 2024-12-07T00:51:40,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700270) roll requested 2024-12-07T00:51:40,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700276 2024-12-07T00:51:40,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700276 2024-12-07T00:51:40,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700276) roll requested 2024-12-07T00:51:40,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700284 2024-12-07T00:51:40,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700284 2024-12-07T00:51:40,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700284) roll requested 2024-12-07T00:51:40,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700292 2024-12-07T00:51:40,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700292 2024-12-07T00:51:40,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700292) roll requested 2024-12-07T00:51:40,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700298 2024-12-07T00:51:40,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700298 2024-12-07T00:51:40,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700298) roll requested 2024-12-07T00:51:40,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700305 2024-12-07T00:51:40,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700305 2024-12-07T00:51:40,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700305) roll requested 2024-12-07T00:51:40,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700313 2024-12-07T00:51:40,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700313 2024-12-07T00:51:40,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700313) roll requested 2024-12-07T00:51:40,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700322 2024-12-07T00:51:40,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700322 2024-12-07T00:51:40,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700322) roll requested 2024-12-07T00:51:40,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700330 2024-12-07T00:51:40,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700330 2024-12-07T00:51:40,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700330) roll requested 2024-12-07T00:51:40,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700337 2024-12-07T00:51:40,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700337 2024-12-07T00:51:40,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700337) roll requested 2024-12-07T00:51:40,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700344 2024-12-07T00:51:40,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700344 2024-12-07T00:51:40,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700344) roll requested 2024-12-07T00:51:40,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700349 2024-12-07T00:51:40,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700349 2024-12-07T00:51:40,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700349) roll requested 2024-12-07T00:51:40,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700358 2024-12-07T00:51:40,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700358 2024-12-07T00:51:40,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700358) roll requested 2024-12-07T00:51:40,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700364 2024-12-07T00:51:40,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700364 2024-12-07T00:51:40,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700364) roll requested 2024-12-07T00:51:40,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700370 2024-12-07T00:51:40,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700370 2024-12-07T00:51:40,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700370) roll requested 2024-12-07T00:51:40,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700375 2024-12-07T00:51:40,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700375 2024-12-07T00:51:40,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700375) roll requested 2024-12-07T00:51:40,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700381 2024-12-07T00:51:40,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700381 2024-12-07T00:51:40,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700381) roll requested 2024-12-07T00:51:40,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700387 2024-12-07T00:51:40,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700387 2024-12-07T00:51:40,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700387) roll requested 2024-12-07T00:51:40,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700392 2024-12-07T00:51:40,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700392 2024-12-07T00:51:40,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700392) roll requested 2024-12-07T00:51:40,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700398 2024-12-07T00:51:40,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700398 2024-12-07T00:51:40,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700398) roll requested 2024-12-07T00:51:40,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700403 2024-12-07T00:51:40,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700403 2024-12-07T00:51:40,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700403) roll requested 2024-12-07T00:51:40,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700409 2024-12-07T00:51:40,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700409 2024-12-07T00:51:40,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700409) roll requested 2024-12-07T00:51:40,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700414 2024-12-07T00:51:40,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700414 2024-12-07T00:51:40,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700414) roll requested 2024-12-07T00:51:40,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700426 2024-12-07T00:51:40,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700426 2024-12-07T00:51:40,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700426) roll requested 2024-12-07T00:51:40,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700439 2024-12-07T00:51:40,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700439 2024-12-07T00:51:40,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700439) roll requested 2024-12-07T00:51:40,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700445 2024-12-07T00:51:40,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700445 2024-12-07T00:51:40,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700445) roll requested 2024-12-07T00:51:40,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700451 2024-12-07T00:51:40,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700451 2024-12-07T00:51:40,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700451) roll requested 2024-12-07T00:51:40,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700458 2024-12-07T00:51:40,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700458 2024-12-07T00:51:40,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700458) roll requested 2024-12-07T00:51:40,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700464 2024-12-07T00:51:40,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700464 2024-12-07T00:51:40,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700464) roll requested 2024-12-07T00:51:40,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700469 2024-12-07T00:51:40,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700469 2024-12-07T00:51:40,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700469) roll requested 2024-12-07T00:51:40,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700474 2024-12-07T00:51:40,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700474 2024-12-07T00:51:40,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700474) roll requested 2024-12-07T00:51:40,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700481 2024-12-07T00:51:40,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700481 2024-12-07T00:51:40,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700481) roll requested 2024-12-07T00:51:40,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700487 2024-12-07T00:51:40,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700487 2024-12-07T00:51:40,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700487) roll requested 2024-12-07T00:51:40,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700493 2024-12-07T00:51:40,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700493 2024-12-07T00:51:40,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700493) roll requested 2024-12-07T00:51:40,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700499 2024-12-07T00:51:40,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700499 2024-12-07T00:51:40,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700499) roll requested 2024-12-07T00:51:40,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700505 2024-12-07T00:51:40,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700505 2024-12-07T00:51:40,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700505) roll requested 2024-12-07T00:51:40,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700511 2024-12-07T00:51:40,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700511 2024-12-07T00:51:40,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700511) roll requested 2024-12-07T00:51:40,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700517 2024-12-07T00:51:40,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700517 2024-12-07T00:51:40,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700517) roll requested 2024-12-07T00:51:40,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700523 2024-12-07T00:51:40,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700523 2024-12-07T00:51:40,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700523) roll requested 2024-12-07T00:51:40,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700528 2024-12-07T00:51:40,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700528 2024-12-07T00:51:40,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700528) roll requested 2024-12-07T00:51:40,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700535 2024-12-07T00:51:40,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700535 2024-12-07T00:51:40,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700535) roll requested 2024-12-07T00:51:40,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700540 2024-12-07T00:51:40,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700540 2024-12-07T00:51:40,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700540) roll requested 2024-12-07T00:51:40,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700546 2024-12-07T00:51:40,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700546 2024-12-07T00:51:40,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700546) roll requested 2024-12-07T00:51:40,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700552 2024-12-07T00:51:40,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700552 2024-12-07T00:51:40,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700552) roll requested 2024-12-07T00:51:40,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700558 2024-12-07T00:51:40,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700558 2024-12-07T00:51:40,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700558) roll requested 2024-12-07T00:51:40,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700564 2024-12-07T00:51:40,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700564 2024-12-07T00:51:40,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700564) roll requested 2024-12-07T00:51:40,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700570 2024-12-07T00:51:40,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700570 2024-12-07T00:51:40,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700570) roll requested 2024-12-07T00:51:40,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700575 2024-12-07T00:51:40,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700575 2024-12-07T00:51:40,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700575) roll requested 2024-12-07T00:51:40,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700581 2024-12-07T00:51:40,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700581 2024-12-07T00:51:40,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700581) roll requested 2024-12-07T00:51:40,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700587 2024-12-07T00:51:40,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700587 2024-12-07T00:51:40,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700587) roll requested 2024-12-07T00:51:40,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700593 2024-12-07T00:51:40,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700593 2024-12-07T00:51:40,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700593) roll requested 2024-12-07T00:51:40,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700599 2024-12-07T00:51:40,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700599 2024-12-07T00:51:40,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700599) roll requested 2024-12-07T00:51:40,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700604 2024-12-07T00:51:40,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700604 2024-12-07T00:51:40,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700604) roll requested 2024-12-07T00:51:40,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700610 2024-12-07T00:51:40,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700610 2024-12-07T00:51:40,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700610) roll requested 2024-12-07T00:51:40,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700616 2024-12-07T00:51:40,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700616 2024-12-07T00:51:40,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700616) roll requested 2024-12-07T00:51:40,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700621 2024-12-07T00:51:40,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700621 2024-12-07T00:51:40,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700621) roll requested 2024-12-07T00:51:40,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700627 2024-12-07T00:51:40,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700627 2024-12-07T00:51:40,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700627) roll requested 2024-12-07T00:51:40,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700633 2024-12-07T00:51:40,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700633 2024-12-07T00:51:40,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700633) roll requested 2024-12-07T00:51:40,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700639 2024-12-07T00:51:40,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700639 2024-12-07T00:51:40,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700639) roll requested 2024-12-07T00:51:40,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700645 2024-12-07T00:51:40,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700645 2024-12-07T00:51:40,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700645) roll requested 2024-12-07T00:51:40,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700651 2024-12-07T00:51:40,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700651 2024-12-07T00:51:40,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700651) roll requested 2024-12-07T00:51:40,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700664 2024-12-07T00:51:40,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700664 2024-12-07T00:51:40,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700664) roll requested 2024-12-07T00:51:40,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700669 2024-12-07T00:51:40,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700669 2024-12-07T00:51:40,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700669) roll requested 2024-12-07T00:51:40,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700675 2024-12-07T00:51:40,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700675 2024-12-07T00:51:40,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700675) roll requested 2024-12-07T00:51:40,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700684 2024-12-07T00:51:40,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700684 2024-12-07T00:51:40,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700684) roll requested 2024-12-07T00:51:40,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700689 2024-12-07T00:51:40,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700689 2024-12-07T00:51:40,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700689) roll requested 2024-12-07T00:51:40,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700695 2024-12-07T00:51:40,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700695 2024-12-07T00:51:40,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700695) roll requested 2024-12-07T00:51:40,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700702 2024-12-07T00:51:40,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700702 2024-12-07T00:51:40,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700702) roll requested 2024-12-07T00:51:40,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700715 2024-12-07T00:51:40,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700715 2024-12-07T00:51:40,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700715) roll requested 2024-12-07T00:51:40,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700731 2024-12-07T00:51:40,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700731 2024-12-07T00:51:40,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700731) roll requested 2024-12-07T00:51:40,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700740 2024-12-07T00:51:40,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700740 2024-12-07T00:51:40,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700740) roll requested 2024-12-07T00:51:40,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700749 2024-12-07T00:51:40,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700749 2024-12-07T00:51:40,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700749) roll requested 2024-12-07T00:51:40,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700757 2024-12-07T00:51:40,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700757 2024-12-07T00:51:40,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700757) roll requested 2024-12-07T00:51:40,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700763 2024-12-07T00:51:40,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700763 2024-12-07T00:51:40,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700763) roll requested 2024-12-07T00:51:40,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700770 2024-12-07T00:51:40,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700770 2024-12-07T00:51:40,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700770) roll requested 2024-12-07T00:51:40,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700776 2024-12-07T00:51:40,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700776 2024-12-07T00:51:40,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700776) roll requested 2024-12-07T00:51:40,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700785 2024-12-07T00:51:40,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700785 2024-12-07T00:51:40,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700785) roll requested 2024-12-07T00:51:40,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700804 2024-12-07T00:51:40,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700804 2024-12-07T00:51:40,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700804) roll requested 2024-12-07T00:51:40,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700821 2024-12-07T00:51:40,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700821 2024-12-07T00:51:40,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700821) roll requested 2024-12-07T00:51:40,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700831 2024-12-07T00:51:40,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700831 2024-12-07T00:51:40,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700831) roll requested 2024-12-07T00:51:40,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700847 2024-12-07T00:51:40,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700847 2024-12-07T00:51:40,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700847) roll requested 2024-12-07T00:51:40,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700857 2024-12-07T00:51:40,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700857 2024-12-07T00:51:40,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700857) roll requested 2024-12-07T00:51:40,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700873 2024-12-07T00:51:40,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700873 2024-12-07T00:51:40,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700873) roll requested 2024-12-07T00:51:40,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700884 2024-12-07T00:51:40,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700884 2024-12-07T00:51:40,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700884) roll requested 2024-12-07T00:51:40,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700893 2024-12-07T00:51:40,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700893 2024-12-07T00:51:40,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700893) roll requested 2024-12-07T00:51:40,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700920 2024-12-07T00:51:40,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700920 2024-12-07T00:51:40,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700920) roll requested 2024-12-07T00:51:40,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700929 2024-12-07T00:51:40,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700929 2024-12-07T00:51:40,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700929) roll requested 2024-12-07T00:51:40,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700940 2024-12-07T00:51:40,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700940 2024-12-07T00:51:40,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700940) roll requested 2024-12-07T00:51:40,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700952 2024-12-07T00:51:40,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700952 2024-12-07T00:51:40,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700952) roll requested 2024-12-07T00:51:40,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700965 2024-12-07T00:51:40,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700965 2024-12-07T00:51:40,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700965) roll requested 2024-12-07T00:51:40,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700972 2024-12-07T00:51:40,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700972 2024-12-07T00:51:40,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700972) roll requested 2024-12-07T00:51:40,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700979 2024-12-07T00:51:40,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700979 2024-12-07T00:51:40,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:40,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:40,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:40,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:40,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700979) roll requested 2024-12-07T00:51:40,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532700986 2024-12-07T00:51:40,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:40,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700986 2024-12-07T00:51:41,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532700986) roll requested 2024-12-07T00:51:41,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701000 2024-12-07T00:51:41,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532700986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701000 2024-12-07T00:51:41,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701000) roll requested 2024-12-07T00:51:41,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701007 2024-12-07T00:51:41,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701007 2024-12-07T00:51:41,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701007) roll requested 2024-12-07T00:51:41,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701013 2024-12-07T00:51:41,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701013 2024-12-07T00:51:41,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701013) roll requested 2024-12-07T00:51:41,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701021 2024-12-07T00:51:41,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701021 2024-12-07T00:51:41,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701021) roll requested 2024-12-07T00:51:41,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701028 2024-12-07T00:51:41,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701028 2024-12-07T00:51:41,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701028) roll requested 2024-12-07T00:51:41,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701036 2024-12-07T00:51:41,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701036 2024-12-07T00:51:41,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701036) roll requested 2024-12-07T00:51:41,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701043 2024-12-07T00:51:41,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701043 2024-12-07T00:51:41,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701043) roll requested 2024-12-07T00:51:41,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701049 2024-12-07T00:51:41,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701049 2024-12-07T00:51:41,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701049) roll requested 2024-12-07T00:51:41,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701059 2024-12-07T00:51:41,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701059 2024-12-07T00:51:41,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701059) roll requested 2024-12-07T00:51:41,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701066 2024-12-07T00:51:41,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701066 2024-12-07T00:51:41,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701066) roll requested 2024-12-07T00:51:41,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701072 2024-12-07T00:51:41,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701072 2024-12-07T00:51:41,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701072) roll requested 2024-12-07T00:51:41,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701079 2024-12-07T00:51:41,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701079 2024-12-07T00:51:41,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701079) roll requested 2024-12-07T00:51:41,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701088 2024-12-07T00:51:41,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701088 2024-12-07T00:51:41,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701088) roll requested 2024-12-07T00:51:41,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701095 2024-12-07T00:51:41,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701095 2024-12-07T00:51:41,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701095) roll requested 2024-12-07T00:51:41,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701104 2024-12-07T00:51:41,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701104 2024-12-07T00:51:41,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701104) roll requested 2024-12-07T00:51:41,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701110 2024-12-07T00:51:41,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701110 2024-12-07T00:51:41,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701110) roll requested 2024-12-07T00:51:41,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701119 2024-12-07T00:51:41,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701119 2024-12-07T00:51:41,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701119) roll requested 2024-12-07T00:51:41,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701126 2024-12-07T00:51:41,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701126 2024-12-07T00:51:41,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701126) roll requested 2024-12-07T00:51:41,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701134 2024-12-07T00:51:41,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701134 2024-12-07T00:51:41,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701134) roll requested 2024-12-07T00:51:41,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701141 2024-12-07T00:51:41,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701141 2024-12-07T00:51:41,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701141) roll requested 2024-12-07T00:51:41,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701148 2024-12-07T00:51:41,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701148 2024-12-07T00:51:41,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701148) roll requested 2024-12-07T00:51:41,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701154 2024-12-07T00:51:41,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701154 2024-12-07T00:51:41,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701154) roll requested 2024-12-07T00:51:41,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701161 2024-12-07T00:51:41,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701161 2024-12-07T00:51:41,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701161) roll requested 2024-12-07T00:51:41,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701168 2024-12-07T00:51:41,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701168 2024-12-07T00:51:41,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701168) roll requested 2024-12-07T00:51:41,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701174 2024-12-07T00:51:41,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701174 2024-12-07T00:51:41,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701174) roll requested 2024-12-07T00:51:41,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701181 2024-12-07T00:51:41,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701181 2024-12-07T00:51:41,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701181) roll requested 2024-12-07T00:51:41,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701188 2024-12-07T00:51:41,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701188 2024-12-07T00:51:41,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701188) roll requested 2024-12-07T00:51:41,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701196 2024-12-07T00:51:41,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701196 2024-12-07T00:51:41,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701196) roll requested 2024-12-07T00:51:41,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701202 2024-12-07T00:51:41,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701202 2024-12-07T00:51:41,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701202) roll requested 2024-12-07T00:51:41,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701210 2024-12-07T00:51:41,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701210 2024-12-07T00:51:41,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701210) roll requested 2024-12-07T00:51:41,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701218 2024-12-07T00:51:41,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701218 2024-12-07T00:51:41,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701218) roll requested 2024-12-07T00:51:41,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701224 2024-12-07T00:51:41,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701224 2024-12-07T00:51:41,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701224) roll requested 2024-12-07T00:51:41,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701232 2024-12-07T00:51:41,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701232 2024-12-07T00:51:41,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701232) roll requested 2024-12-07T00:51:41,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701239 2024-12-07T00:51:41,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701239 2024-12-07T00:51:41,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701239) roll requested 2024-12-07T00:51:41,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701246 2024-12-07T00:51:41,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701246 2024-12-07T00:51:41,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701246) roll requested 2024-12-07T00:51:41,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701252 2024-12-07T00:51:41,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701252 2024-12-07T00:51:41,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701252) roll requested 2024-12-07T00:51:41,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701259 2024-12-07T00:51:41,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701259 2024-12-07T00:51:41,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701259) roll requested 2024-12-07T00:51:41,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701265 2024-12-07T00:51:41,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701265 2024-12-07T00:51:41,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701265) roll requested 2024-12-07T00:51:41,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701271 2024-12-07T00:51:41,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701271 2024-12-07T00:51:41,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701271) roll requested 2024-12-07T00:51:41,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701278 2024-12-07T00:51:41,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701278 2024-12-07T00:51:41,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701278) roll requested 2024-12-07T00:51:41,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701285 2024-12-07T00:51:41,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701285 2024-12-07T00:51:41,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701285) roll requested 2024-12-07T00:51:41,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701293 2024-12-07T00:51:41,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701293 2024-12-07T00:51:41,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701293) roll requested 2024-12-07T00:51:41,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701299 2024-12-07T00:51:41,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701299 2024-12-07T00:51:41,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701299) roll requested 2024-12-07T00:51:41,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701304 2024-12-07T00:51:41,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701304 2024-12-07T00:51:41,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701304) roll requested 2024-12-07T00:51:41,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701311 2024-12-07T00:51:41,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701311 2024-12-07T00:51:41,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701311) roll requested 2024-12-07T00:51:41,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701318 2024-12-07T00:51:41,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701318 2024-12-07T00:51:41,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701318) roll requested 2024-12-07T00:51:41,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701325 2024-12-07T00:51:41,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701325 2024-12-07T00:51:41,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701325) roll requested 2024-12-07T00:51:41,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701334 2024-12-07T00:51:41,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701334 2024-12-07T00:51:41,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701334) roll requested 2024-12-07T00:51:41,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701342 2024-12-07T00:51:41,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701342 2024-12-07T00:51:41,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701342) roll requested 2024-12-07T00:51:41,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701353 2024-12-07T00:51:41,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701353 2024-12-07T00:51:41,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701353) roll requested 2024-12-07T00:51:41,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701360 2024-12-07T00:51:41,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701360 2024-12-07T00:51:41,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701360) roll requested 2024-12-07T00:51:41,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701367 2024-12-07T00:51:41,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701367 2024-12-07T00:51:41,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701367) roll requested 2024-12-07T00:51:41,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701373 2024-12-07T00:51:41,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701373 2024-12-07T00:51:41,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701373) roll requested 2024-12-07T00:51:41,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701380 2024-12-07T00:51:41,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701380 2024-12-07T00:51:41,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701380) roll requested 2024-12-07T00:51:41,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701387 2024-12-07T00:51:41,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701387 2024-12-07T00:51:41,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701387) roll requested 2024-12-07T00:51:41,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701394 2024-12-07T00:51:41,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701394 2024-12-07T00:51:41,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701394) roll requested 2024-12-07T00:51:41,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701401 2024-12-07T00:51:41,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701401 2024-12-07T00:51:41,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701401) roll requested 2024-12-07T00:51:41,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701408 2024-12-07T00:51:41,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701408 2024-12-07T00:51:41,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701408) roll requested 2024-12-07T00:51:41,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701415 2024-12-07T00:51:41,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701415 2024-12-07T00:51:41,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701415) roll requested 2024-12-07T00:51:41,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701424 2024-12-07T00:51:41,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701424 2024-12-07T00:51:41,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701424) roll requested 2024-12-07T00:51:41,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701430 2024-12-07T00:51:41,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701430 2024-12-07T00:51:41,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701430) roll requested 2024-12-07T00:51:41,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701438 2024-12-07T00:51:41,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701438 2024-12-07T00:51:41,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701438) roll requested 2024-12-07T00:51:41,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701445 2024-12-07T00:51:41,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701445 2024-12-07T00:51:41,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701445) roll requested 2024-12-07T00:51:41,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701451 2024-12-07T00:51:41,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701451 2024-12-07T00:51:41,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701451) roll requested 2024-12-07T00:51:41,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701459 2024-12-07T00:51:41,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701459 2024-12-07T00:51:41,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701459) roll requested 2024-12-07T00:51:41,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701466 2024-12-07T00:51:41,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701466 2024-12-07T00:51:41,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701466) roll requested 2024-12-07T00:51:41,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701473 2024-12-07T00:51:41,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701473 2024-12-07T00:51:41,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701473) roll requested 2024-12-07T00:51:41,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701479 2024-12-07T00:51:41,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701479 2024-12-07T00:51:41,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701479) roll requested 2024-12-07T00:51:41,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701486 2024-12-07T00:51:41,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701486 2024-12-07T00:51:41,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701486) roll requested 2024-12-07T00:51:41,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701492 2024-12-07T00:51:41,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701492 2024-12-07T00:51:41,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701492) roll requested 2024-12-07T00:51:41,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701501 2024-12-07T00:51:41,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701501 2024-12-07T00:51:41,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701501) roll requested 2024-12-07T00:51:41,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701509 2024-12-07T00:51:41,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701509 2024-12-07T00:51:41,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701509) roll requested 2024-12-07T00:51:41,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701518 2024-12-07T00:51:41,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701518 2024-12-07T00:51:41,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701518) roll requested 2024-12-07T00:51:41,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701525 2024-12-07T00:51:41,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701525 2024-12-07T00:51:41,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701525) roll requested 2024-12-07T00:51:41,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701531 2024-12-07T00:51:41,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701531 2024-12-07T00:51:41,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701531) roll requested 2024-12-07T00:51:41,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701537 2024-12-07T00:51:41,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701537 2024-12-07T00:51:41,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701537) roll requested 2024-12-07T00:51:41,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701544 2024-12-07T00:51:41,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701544 2024-12-07T00:51:41,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701544) roll requested 2024-12-07T00:51:41,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701550 2024-12-07T00:51:41,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701550 2024-12-07T00:51:41,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701550) roll requested 2024-12-07T00:51:41,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701556 2024-12-07T00:51:41,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701556 2024-12-07T00:51:41,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701556) roll requested 2024-12-07T00:51:41,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701564 2024-12-07T00:51:41,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701564 2024-12-07T00:51:41,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701564) roll requested 2024-12-07T00:51:41,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701570 2024-12-07T00:51:41,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701570 2024-12-07T00:51:41,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701570) roll requested 2024-12-07T00:51:41,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701576 2024-12-07T00:51:41,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701576 2024-12-07T00:51:41,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701576) roll requested 2024-12-07T00:51:41,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701582 2024-12-07T00:51:41,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701582 2024-12-07T00:51:41,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701582) roll requested 2024-12-07T00:51:41,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701588 2024-12-07T00:51:41,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701588 2024-12-07T00:51:41,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701588) roll requested 2024-12-07T00:51:41,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701595 2024-12-07T00:51:41,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701595 2024-12-07T00:51:41,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701595) roll requested 2024-12-07T00:51:41,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701601 2024-12-07T00:51:41,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701601 2024-12-07T00:51:41,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701601) roll requested 2024-12-07T00:51:41,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701606 2024-12-07T00:51:41,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701606 2024-12-07T00:51:41,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701606) roll requested 2024-12-07T00:51:41,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701612 2024-12-07T00:51:41,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701612 2024-12-07T00:51:41,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701612) roll requested 2024-12-07T00:51:41,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701619 2024-12-07T00:51:41,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701619 2024-12-07T00:51:41,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701619) roll requested 2024-12-07T00:51:41,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701626 2024-12-07T00:51:41,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701626 2024-12-07T00:51:41,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701626) roll requested 2024-12-07T00:51:41,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701632 2024-12-07T00:51:41,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701632 2024-12-07T00:51:41,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701632) roll requested 2024-12-07T00:51:41,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701639 2024-12-07T00:51:41,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701639 2024-12-07T00:51:41,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701639) roll requested 2024-12-07T00:51:41,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701648 2024-12-07T00:51:41,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701648 2024-12-07T00:51:41,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701648) roll requested 2024-12-07T00:51:41,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701660 2024-12-07T00:51:41,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701660 2024-12-07T00:51:41,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701660) roll requested 2024-12-07T00:51:41,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701666 2024-12-07T00:51:41,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701666 2024-12-07T00:51:41,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701666) roll requested 2024-12-07T00:51:41,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701671 2024-12-07T00:51:41,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701671 2024-12-07T00:51:41,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701671) roll requested 2024-12-07T00:51:41,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701677 2024-12-07T00:51:41,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701677 2024-12-07T00:51:41,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701677) roll requested 2024-12-07T00:51:41,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701684 2024-12-07T00:51:41,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701684 2024-12-07T00:51:41,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701684) roll requested 2024-12-07T00:51:41,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701690 2024-12-07T00:51:41,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701690 2024-12-07T00:51:41,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701690) roll requested 2024-12-07T00:51:41,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701695 2024-12-07T00:51:41,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701695 2024-12-07T00:51:41,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701695) roll requested 2024-12-07T00:51:41,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701701 2024-12-07T00:51:41,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701701 2024-12-07T00:51:41,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701701) roll requested 2024-12-07T00:51:41,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701709 2024-12-07T00:51:41,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701709 2024-12-07T00:51:41,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701709) roll requested 2024-12-07T00:51:41,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701716 2024-12-07T00:51:41,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701716 2024-12-07T00:51:41,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701716) roll requested 2024-12-07T00:51:41,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701721 2024-12-07T00:51:41,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701721 2024-12-07T00:51:41,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701721) roll requested 2024-12-07T00:51:41,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701728 2024-12-07T00:51:41,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701728 2024-12-07T00:51:41,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701728) roll requested 2024-12-07T00:51:41,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701738 2024-12-07T00:51:41,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701738 2024-12-07T00:51:41,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701738) roll requested 2024-12-07T00:51:41,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701749 2024-12-07T00:51:41,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701749 2024-12-07T00:51:41,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701749) roll requested 2024-12-07T00:51:41,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701756 2024-12-07T00:51:41,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701756 2024-12-07T00:51:41,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701756) roll requested 2024-12-07T00:51:41,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701763 2024-12-07T00:51:41,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701763 2024-12-07T00:51:41,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701763) roll requested 2024-12-07T00:51:41,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701769 2024-12-07T00:51:41,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701769 2024-12-07T00:51:41,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701769) roll requested 2024-12-07T00:51:41,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701775 2024-12-07T00:51:41,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701775 2024-12-07T00:51:41,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701775) roll requested 2024-12-07T00:51:41,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701783 2024-12-07T00:51:41,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701783 2024-12-07T00:51:41,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701783) roll requested 2024-12-07T00:51:41,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701790 2024-12-07T00:51:41,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701790 2024-12-07T00:51:41,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701790) roll requested 2024-12-07T00:51:41,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701796 2024-12-07T00:51:41,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701796 2024-12-07T00:51:41,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701796) roll requested 2024-12-07T00:51:41,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701802 2024-12-07T00:51:41,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701802 2024-12-07T00:51:41,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701802) roll requested 2024-12-07T00:51:41,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701810 2024-12-07T00:51:41,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701810 2024-12-07T00:51:41,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701810) roll requested 2024-12-07T00:51:41,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701816 2024-12-07T00:51:41,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701816 2024-12-07T00:51:41,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701816) roll requested 2024-12-07T00:51:41,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701824 2024-12-07T00:51:41,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701824 2024-12-07T00:51:41,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701824) roll requested 2024-12-07T00:51:41,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701831 2024-12-07T00:51:41,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701831 2024-12-07T00:51:41,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701831) roll requested 2024-12-07T00:51:41,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701839 2024-12-07T00:51:41,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701839 2024-12-07T00:51:41,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701839) roll requested 2024-12-07T00:51:41,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701845 2024-12-07T00:51:41,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701845 2024-12-07T00:51:41,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701845) roll requested 2024-12-07T00:51:41,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701851 2024-12-07T00:51:41,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701851 2024-12-07T00:51:41,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701851) roll requested 2024-12-07T00:51:41,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701858 2024-12-07T00:51:41,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701858 2024-12-07T00:51:41,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701858) roll requested 2024-12-07T00:51:41,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701866 2024-12-07T00:51:41,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701866 2024-12-07T00:51:41,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701866) roll requested 2024-12-07T00:51:41,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701874 2024-12-07T00:51:41,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701874 2024-12-07T00:51:41,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701874) roll requested 2024-12-07T00:51:41,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701881 2024-12-07T00:51:41,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701881 2024-12-07T00:51:41,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701881) roll requested 2024-12-07T00:51:41,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701888 2024-12-07T00:51:41,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701888 2024-12-07T00:51:41,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701888) roll requested 2024-12-07T00:51:41,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701894 2024-12-07T00:51:41,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701894 2024-12-07T00:51:41,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701894) roll requested 2024-12-07T00:51:41,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701900 2024-12-07T00:51:41,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701900 2024-12-07T00:51:41,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701900) roll requested 2024-12-07T00:51:41,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701908 2024-12-07T00:51:41,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701908 2024-12-07T00:51:41,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701908) roll requested 2024-12-07T00:51:41,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701914 2024-12-07T00:51:41,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701914 2024-12-07T00:51:41,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701914) roll requested 2024-12-07T00:51:41,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701923 2024-12-07T00:51:41,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701923 2024-12-07T00:51:41,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701923) roll requested 2024-12-07T00:51:41,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701930 2024-12-07T00:51:41,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701930 2024-12-07T00:51:41,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701930) roll requested 2024-12-07T00:51:41,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701937 2024-12-07T00:51:41,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701937 2024-12-07T00:51:41,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701937) roll requested 2024-12-07T00:51:41,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701943 2024-12-07T00:51:41,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701943 2024-12-07T00:51:41,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701943) roll requested 2024-12-07T00:51:41,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701948 2024-12-07T00:51:41,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701948 2024-12-07T00:51:41,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701948) roll requested 2024-12-07T00:51:41,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701954 2024-12-07T00:51:41,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701954 2024-12-07T00:51:41,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701954) roll requested 2024-12-07T00:51:41,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701963 2024-12-07T00:51:41,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701963 2024-12-07T00:51:41,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701963) roll requested 2024-12-07T00:51:41,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701969 2024-12-07T00:51:41,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701969 2024-12-07T00:51:41,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701969) roll requested 2024-12-07T00:51:41,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701975 2024-12-07T00:51:41,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701975 2024-12-07T00:51:41,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701975) roll requested 2024-12-07T00:51:41,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701981 2024-12-07T00:51:41,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701981 2024-12-07T00:51:41,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701981) roll requested 2024-12-07T00:51:41,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701987 2024-12-07T00:51:41,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:41,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701987 2024-12-07T00:51:41,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:41,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:41,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:41,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:41,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701987) roll requested 2024-12-07T00:51:41,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532701994 2024-12-07T00:51:42,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701994 2024-12-07T00:51:42,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532701994) roll requested 2024-12-07T00:51:42,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702002 2024-12-07T00:51:42,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532701994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702002 2024-12-07T00:51:42,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702002) roll requested 2024-12-07T00:51:42,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702010 2024-12-07T00:51:42,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702010 2024-12-07T00:51:42,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702010) roll requested 2024-12-07T00:51:42,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702021 2024-12-07T00:51:42,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702021 2024-12-07T00:51:42,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702021) roll requested 2024-12-07T00:51:42,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702028 2024-12-07T00:51:42,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702028 2024-12-07T00:51:42,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702028) roll requested 2024-12-07T00:51:42,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702033 2024-12-07T00:51:42,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702033 2024-12-07T00:51:42,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702033) roll requested 2024-12-07T00:51:42,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702039 2024-12-07T00:51:42,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702039 2024-12-07T00:51:42,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702039) roll requested 2024-12-07T00:51:42,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702046 2024-12-07T00:51:42,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702046 2024-12-07T00:51:42,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702046) roll requested 2024-12-07T00:51:42,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702054 2024-12-07T00:51:42,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702054 2024-12-07T00:51:42,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702054) roll requested 2024-12-07T00:51:42,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702066 2024-12-07T00:51:42,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702066 2024-12-07T00:51:42,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702066) roll requested 2024-12-07T00:51:42,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702072 2024-12-07T00:51:42,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702072 2024-12-07T00:51:42,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702072) roll requested 2024-12-07T00:51:42,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702078 2024-12-07T00:51:42,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702078 2024-12-07T00:51:42,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702078) roll requested 2024-12-07T00:51:42,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702084 2024-12-07T00:51:42,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702084 2024-12-07T00:51:42,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702084) roll requested 2024-12-07T00:51:42,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702089 2024-12-07T00:51:42,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702089 2024-12-07T00:51:42,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702089) roll requested 2024-12-07T00:51:42,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702095 2024-12-07T00:51:42,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702095 2024-12-07T00:51:42,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702095) roll requested 2024-12-07T00:51:42,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702101 2024-12-07T00:51:42,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702101 2024-12-07T00:51:42,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702101) roll requested 2024-12-07T00:51:42,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702107 2024-12-07T00:51:42,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702107 2024-12-07T00:51:42,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702107) roll requested 2024-12-07T00:51:42,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702113 2024-12-07T00:51:42,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702113 2024-12-07T00:51:42,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702113) roll requested 2024-12-07T00:51:42,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702119 2024-12-07T00:51:42,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702119 2024-12-07T00:51:42,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702119) roll requested 2024-12-07T00:51:42,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702125 2024-12-07T00:51:42,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702125 2024-12-07T00:51:42,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702125) roll requested 2024-12-07T00:51:42,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702132 2024-12-07T00:51:42,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702132 2024-12-07T00:51:42,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702132) roll requested 2024-12-07T00:51:42,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702139 2024-12-07T00:51:42,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702139 2024-12-07T00:51:42,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702139) roll requested 2024-12-07T00:51:42,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702146 2024-12-07T00:51:42,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702146 2024-12-07T00:51:42,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702146) roll requested 2024-12-07T00:51:42,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702153 2024-12-07T00:51:42,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702153 2024-12-07T00:51:42,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702153) roll requested 2024-12-07T00:51:42,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702159 2024-12-07T00:51:42,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702159 2024-12-07T00:51:42,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702159) roll requested 2024-12-07T00:51:42,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702165 2024-12-07T00:51:42,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702165 2024-12-07T00:51:42,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702165) roll requested 2024-12-07T00:51:42,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702171 2024-12-07T00:51:42,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702171 2024-12-07T00:51:42,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702171) roll requested 2024-12-07T00:51:42,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702178 2024-12-07T00:51:42,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702178 2024-12-07T00:51:42,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702178) roll requested 2024-12-07T00:51:42,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702184 2024-12-07T00:51:42,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702184 2024-12-07T00:51:42,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702184) roll requested 2024-12-07T00:51:42,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702190 2024-12-07T00:51:42,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702190 2024-12-07T00:51:42,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702190) roll requested 2024-12-07T00:51:42,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702200 2024-12-07T00:51:42,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702200 2024-12-07T00:51:42,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702200) roll requested 2024-12-07T00:51:42,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702207 2024-12-07T00:51:42,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702207 2024-12-07T00:51:42,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702207) roll requested 2024-12-07T00:51:42,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702214 2024-12-07T00:51:42,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702214 2024-12-07T00:51:42,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702214) roll requested 2024-12-07T00:51:42,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702222 2024-12-07T00:51:42,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702222 2024-12-07T00:51:42,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702222) roll requested 2024-12-07T00:51:42,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702229 2024-12-07T00:51:42,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702229 2024-12-07T00:51:42,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702229) roll requested 2024-12-07T00:51:42,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702236 2024-12-07T00:51:42,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702236 2024-12-07T00:51:42,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702236) roll requested 2024-12-07T00:51:42,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702242 2024-12-07T00:51:42,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702242 2024-12-07T00:51:42,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702242) roll requested 2024-12-07T00:51:42,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702247 2024-12-07T00:51:42,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702247 2024-12-07T00:51:42,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702247) roll requested 2024-12-07T00:51:42,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702253 2024-12-07T00:51:42,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702253 2024-12-07T00:51:42,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702253) roll requested 2024-12-07T00:51:42,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702259 2024-12-07T00:51:42,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702259 2024-12-07T00:51:42,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702259) roll requested 2024-12-07T00:51:42,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702266 2024-12-07T00:51:42,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702266 2024-12-07T00:51:42,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702266) roll requested 2024-12-07T00:51:42,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702272 2024-12-07T00:51:42,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702272 2024-12-07T00:51:42,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702272) roll requested 2024-12-07T00:51:42,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702278 2024-12-07T00:51:42,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702278 2024-12-07T00:51:42,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702278) roll requested 2024-12-07T00:51:42,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702284 2024-12-07T00:51:42,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702284 2024-12-07T00:51:42,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702284) roll requested 2024-12-07T00:51:42,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702291 2024-12-07T00:51:42,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702291 2024-12-07T00:51:42,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702291) roll requested 2024-12-07T00:51:42,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702300 2024-12-07T00:51:42,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702300 2024-12-07T00:51:42,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702300) roll requested 2024-12-07T00:51:42,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702307 2024-12-07T00:51:42,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702307 2024-12-07T00:51:42,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702307) roll requested 2024-12-07T00:51:42,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702314 2024-12-07T00:51:42,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702314 2024-12-07T00:51:42,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702314) roll requested 2024-12-07T00:51:42,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702322 2024-12-07T00:51:42,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702322 2024-12-07T00:51:42,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702322) roll requested 2024-12-07T00:51:42,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702329 2024-12-07T00:51:42,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702329 2024-12-07T00:51:42,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702329) roll requested 2024-12-07T00:51:42,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702336 2024-12-07T00:51:42,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702336 2024-12-07T00:51:42,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702336) roll requested 2024-12-07T00:51:42,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702343 2024-12-07T00:51:42,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702343 2024-12-07T00:51:42,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702343) roll requested 2024-12-07T00:51:42,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702350 2024-12-07T00:51:42,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702350 2024-12-07T00:51:42,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702350) roll requested 2024-12-07T00:51:42,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702357 2024-12-07T00:51:42,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702357 2024-12-07T00:51:42,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702357) roll requested 2024-12-07T00:51:42,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702363 2024-12-07T00:51:42,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702363 2024-12-07T00:51:42,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702363) roll requested 2024-12-07T00:51:42,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702369 2024-12-07T00:51:42,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702369 2024-12-07T00:51:42,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702369) roll requested 2024-12-07T00:51:42,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702375 2024-12-07T00:51:42,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702375 2024-12-07T00:51:42,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702375) roll requested 2024-12-07T00:51:42,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702382 2024-12-07T00:51:42,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702382 2024-12-07T00:51:42,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702382) roll requested 2024-12-07T00:51:42,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702388 2024-12-07T00:51:42,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702388 2024-12-07T00:51:42,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702388) roll requested 2024-12-07T00:51:42,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702394 2024-12-07T00:51:42,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702394 2024-12-07T00:51:42,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702394) roll requested 2024-12-07T00:51:42,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702401 2024-12-07T00:51:42,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702401 2024-12-07T00:51:42,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702401) roll requested 2024-12-07T00:51:42,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702407 2024-12-07T00:51:42,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702407 2024-12-07T00:51:42,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702407) roll requested 2024-12-07T00:51:42,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702414 2024-12-07T00:51:42,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702414 2024-12-07T00:51:42,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702414) roll requested 2024-12-07T00:51:42,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702420 2024-12-07T00:51:42,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702420 2024-12-07T00:51:42,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702420) roll requested 2024-12-07T00:51:42,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702428 2024-12-07T00:51:42,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702428 2024-12-07T00:51:42,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702428) roll requested 2024-12-07T00:51:42,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702435 2024-12-07T00:51:42,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702435 2024-12-07T00:51:42,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702435) roll requested 2024-12-07T00:51:42,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702443 2024-12-07T00:51:42,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702443 2024-12-07T00:51:42,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702443) roll requested 2024-12-07T00:51:42,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702450 2024-12-07T00:51:42,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702450 2024-12-07T00:51:42,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702450) roll requested 2024-12-07T00:51:42,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702457 2024-12-07T00:51:42,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702457 2024-12-07T00:51:42,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702457) roll requested 2024-12-07T00:51:42,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702463 2024-12-07T00:51:42,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702463 2024-12-07T00:51:42,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702463) roll requested 2024-12-07T00:51:42,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702472 2024-12-07T00:51:42,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702472 2024-12-07T00:51:42,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702472) roll requested 2024-12-07T00:51:42,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702479 2024-12-07T00:51:42,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702479 2024-12-07T00:51:42,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702479) roll requested 2024-12-07T00:51:42,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702487 2024-12-07T00:51:42,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702487 2024-12-07T00:51:42,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702487) roll requested 2024-12-07T00:51:42,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702495 2024-12-07T00:51:42,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702495 2024-12-07T00:51:42,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702495) roll requested 2024-12-07T00:51:42,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702503 2024-12-07T00:51:42,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702503 2024-12-07T00:51:42,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702503) roll requested 2024-12-07T00:51:42,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702510 2024-12-07T00:51:42,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702510 2024-12-07T00:51:42,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702510) roll requested 2024-12-07T00:51:42,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702517 2024-12-07T00:51:42,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702517 2024-12-07T00:51:42,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702517) roll requested 2024-12-07T00:51:42,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702528 2024-12-07T00:51:42,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702528 2024-12-07T00:51:42,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702528) roll requested 2024-12-07T00:51:42,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702535 2024-12-07T00:51:42,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702535 2024-12-07T00:51:42,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702535) roll requested 2024-12-07T00:51:42,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702541 2024-12-07T00:51:42,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702541 2024-12-07T00:51:42,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702541) roll requested 2024-12-07T00:51:42,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702548 2024-12-07T00:51:42,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702548 2024-12-07T00:51:42,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702548) roll requested 2024-12-07T00:51:42,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702554 2024-12-07T00:51:42,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702554 2024-12-07T00:51:42,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702554) roll requested 2024-12-07T00:51:42,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702560 2024-12-07T00:51:42,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702560 2024-12-07T00:51:42,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702560) roll requested 2024-12-07T00:51:42,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702566 2024-12-07T00:51:42,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702566 2024-12-07T00:51:42,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702566) roll requested 2024-12-07T00:51:42,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702573 2024-12-07T00:51:42,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702573 2024-12-07T00:51:42,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702573) roll requested 2024-12-07T00:51:42,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702579 2024-12-07T00:51:42,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702579 2024-12-07T00:51:42,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702579) roll requested 2024-12-07T00:51:42,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702589 2024-12-07T00:51:42,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702589 2024-12-07T00:51:42,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702589) roll requested 2024-12-07T00:51:42,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702597 2024-12-07T00:51:42,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702597 2024-12-07T00:51:42,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702597) roll requested 2024-12-07T00:51:42,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702604 2024-12-07T00:51:42,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702604 2024-12-07T00:51:42,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702604) roll requested 2024-12-07T00:51:42,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702611 2024-12-07T00:51:42,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702611 2024-12-07T00:51:42,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702611) roll requested 2024-12-07T00:51:42,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702618 2024-12-07T00:51:42,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702618 2024-12-07T00:51:42,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702618) roll requested 2024-12-07T00:51:42,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702624 2024-12-07T00:51:42,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702624 2024-12-07T00:51:42,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702624) roll requested 2024-12-07T00:51:42,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702630 2024-12-07T00:51:42,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702630 2024-12-07T00:51:42,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702630) roll requested 2024-12-07T00:51:42,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702640 2024-12-07T00:51:42,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702640 2024-12-07T00:51:42,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702640) roll requested 2024-12-07T00:51:42,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702646 2024-12-07T00:51:42,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702646 2024-12-07T00:51:42,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702646) roll requested 2024-12-07T00:51:42,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702652 2024-12-07T00:51:42,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702652 2024-12-07T00:51:42,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702652) roll requested 2024-12-07T00:51:42,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702663 2024-12-07T00:51:42,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702663 2024-12-07T00:51:42,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702663) roll requested 2024-12-07T00:51:42,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702670 2024-12-07T00:51:42,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702670 2024-12-07T00:51:42,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702670) roll requested 2024-12-07T00:51:42,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702676 2024-12-07T00:51:42,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702676 2024-12-07T00:51:42,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702676) roll requested 2024-12-07T00:51:42,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702682 2024-12-07T00:51:42,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702682 2024-12-07T00:51:42,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702682) roll requested 2024-12-07T00:51:42,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702689 2024-12-07T00:51:42,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702689 2024-12-07T00:51:42,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702689) roll requested 2024-12-07T00:51:42,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702695 2024-12-07T00:51:42,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702695 2024-12-07T00:51:42,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702695) roll requested 2024-12-07T00:51:42,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702702 2024-12-07T00:51:42,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702702 2024-12-07T00:51:42,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702702) roll requested 2024-12-07T00:51:42,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702710 2024-12-07T00:51:42,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702710 2024-12-07T00:51:42,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702710) roll requested 2024-12-07T00:51:42,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702718 2024-12-07T00:51:42,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702718 2024-12-07T00:51:42,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702718) roll requested 2024-12-07T00:51:42,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702725 2024-12-07T00:51:42,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702725 2024-12-07T00:51:42,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702725) roll requested 2024-12-07T00:51:42,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702732 2024-12-07T00:51:42,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702732 2024-12-07T00:51:42,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702732) roll requested 2024-12-07T00:51:42,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702741 2024-12-07T00:51:42,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702741 2024-12-07T00:51:42,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702741) roll requested 2024-12-07T00:51:42,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702747 2024-12-07T00:51:42,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702747 2024-12-07T00:51:42,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702747) roll requested 2024-12-07T00:51:42,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702753 2024-12-07T00:51:42,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702753 2024-12-07T00:51:42,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702753) roll requested 2024-12-07T00:51:42,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702760 2024-12-07T00:51:42,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702760 2024-12-07T00:51:42,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702760) roll requested 2024-12-07T00:51:42,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702766 2024-12-07T00:51:42,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702766 2024-12-07T00:51:42,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702766) roll requested 2024-12-07T00:51:42,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702772 2024-12-07T00:51:42,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702772 2024-12-07T00:51:42,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702772) roll requested 2024-12-07T00:51:42,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702780 2024-12-07T00:51:42,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702780 2024-12-07T00:51:42,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702780) roll requested 2024-12-07T00:51:42,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702787 2024-12-07T00:51:42,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702787 2024-12-07T00:51:42,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702787) roll requested 2024-12-07T00:51:42,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702794 2024-12-07T00:51:42,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702794 2024-12-07T00:51:42,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702794) roll requested 2024-12-07T00:51:42,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702800 2024-12-07T00:51:42,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702800 2024-12-07T00:51:42,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702800) roll requested 2024-12-07T00:51:42,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702808 2024-12-07T00:51:42,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702808 2024-12-07T00:51:42,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702808) roll requested 2024-12-07T00:51:42,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702816 2024-12-07T00:51:42,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702816 2024-12-07T00:51:42,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702816) roll requested 2024-12-07T00:51:42,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702823 2024-12-07T00:51:42,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702823 2024-12-07T00:51:42,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702823) roll requested 2024-12-07T00:51:42,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702831 2024-12-07T00:51:42,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702831 2024-12-07T00:51:42,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702831) roll requested 2024-12-07T00:51:42,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702837 2024-12-07T00:51:42,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702837 2024-12-07T00:51:42,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702837) roll requested 2024-12-07T00:51:42,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702843 2024-12-07T00:51:42,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702843 2024-12-07T00:51:42,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702843) roll requested 2024-12-07T00:51:42,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702850 2024-12-07T00:51:42,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702850 2024-12-07T00:51:42,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702850) roll requested 2024-12-07T00:51:42,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702856 2024-12-07T00:51:42,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702856 2024-12-07T00:51:42,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702856) roll requested 2024-12-07T00:51:42,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702863 2024-12-07T00:51:42,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702863 2024-12-07T00:51:42,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702863) roll requested 2024-12-07T00:51:42,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702871 2024-12-07T00:51:42,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702871 2024-12-07T00:51:42,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702871) roll requested 2024-12-07T00:51:42,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702878 2024-12-07T00:51:42,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702878 2024-12-07T00:51:42,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702878) roll requested 2024-12-07T00:51:42,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702885 2024-12-07T00:51:42,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702885 2024-12-07T00:51:42,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702885) roll requested 2024-12-07T00:51:42,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702891 2024-12-07T00:51:42,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702891 2024-12-07T00:51:42,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702891) roll requested 2024-12-07T00:51:42,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702898 2024-12-07T00:51:42,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702898 2024-12-07T00:51:42,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702898) roll requested 2024-12-07T00:51:42,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702906 2024-12-07T00:51:42,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702906 2024-12-07T00:51:42,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702906) roll requested 2024-12-07T00:51:42,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702913 2024-12-07T00:51:42,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702913 2024-12-07T00:51:42,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702913) roll requested 2024-12-07T00:51:42,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702920 2024-12-07T00:51:42,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702920 2024-12-07T00:51:42,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702920) roll requested 2024-12-07T00:51:42,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702927 2024-12-07T00:51:42,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702927 2024-12-07T00:51:42,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702927) roll requested 2024-12-07T00:51:42,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702934 2024-12-07T00:51:42,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702934 2024-12-07T00:51:42,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702934) roll requested 2024-12-07T00:51:42,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702939 2024-12-07T00:51:42,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702939 2024-12-07T00:51:42,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702939) roll requested 2024-12-07T00:51:42,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702945 2024-12-07T00:51:42,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702945 2024-12-07T00:51:42,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702945) roll requested 2024-12-07T00:51:42,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702951 2024-12-07T00:51:42,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702951 2024-12-07T00:51:42,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702951) roll requested 2024-12-07T00:51:42,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702959 2024-12-07T00:51:42,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702959 2024-12-07T00:51:42,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702959) roll requested 2024-12-07T00:51:42,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702967 2024-12-07T00:51:42,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702967 2024-12-07T00:51:42,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702967) roll requested 2024-12-07T00:51:42,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702972 2024-12-07T00:51:42,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702972 2024-12-07T00:51:42,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702972) roll requested 2024-12-07T00:51:42,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702978 2024-12-07T00:51:42,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702978 2024-12-07T00:51:42,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702978) roll requested 2024-12-07T00:51:42,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702984 2024-12-07T00:51:42,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702984 2024-12-07T00:51:42,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702984) roll requested 2024-12-07T00:51:42,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702991 2024-12-07T00:51:42,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:42,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702991 2024-12-07T00:51:42,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:42,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:42,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:42,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:42,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702991) roll requested 2024-12-07T00:51:42,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532702998 2024-12-07T00:51:43,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702998 2024-12-07T00:51:43,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532702998) roll requested 2024-12-07T00:51:43,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703005 2024-12-07T00:51:43,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532702998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703005 2024-12-07T00:51:43,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703005) roll requested 2024-12-07T00:51:43,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703012 2024-12-07T00:51:43,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703012 2024-12-07T00:51:43,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703012) roll requested 2024-12-07T00:51:43,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703018 2024-12-07T00:51:43,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703018 2024-12-07T00:51:43,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703012 is not closed yet, will try archiving it next time 2024-12-07T00:51:43,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703018) roll requested 2024-12-07T00:51:43,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703025 2024-12-07T00:51:43,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703025 2024-12-07T00:51:43,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703025) roll requested 2024-12-07T00:51:43,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703032 2024-12-07T00:51:43,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703032 2024-12-07T00:51:43,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703032) roll requested 2024-12-07T00:51:43,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703039 2024-12-07T00:51:43,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703039 2024-12-07T00:51:43,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703039) roll requested 2024-12-07T00:51:43,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703047 2024-12-07T00:51:43,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703047 2024-12-07T00:51:43,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,053 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,053 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703047) roll requested 2024-12-07T00:51:43,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703053 2024-12-07T00:51:43,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703053 2024-12-07T00:51:43,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703053) roll requested 2024-12-07T00:51:43,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703061 2024-12-07T00:51:43,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703061 2024-12-07T00:51:43,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703061) roll requested 2024-12-07T00:51:43,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703068 2024-12-07T00:51:43,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703068 2024-12-07T00:51:43,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703068) roll requested 2024-12-07T00:51:43,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703074 2024-12-07T00:51:43,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703074 2024-12-07T00:51:43,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703074) roll requested 2024-12-07T00:51:43,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703080 2024-12-07T00:51:43,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703080 2024-12-07T00:51:43,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,088 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703080) roll requested 2024-12-07T00:51:43,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703088 2024-12-07T00:51:43,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703088 2024-12-07T00:51:43,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703088) roll requested 2024-12-07T00:51:43,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703094 2024-12-07T00:51:43,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703094 2024-12-07T00:51:43,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703094) roll requested 2024-12-07T00:51:43,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703102 2024-12-07T00:51:43,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703102 2024-12-07T00:51:43,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703102) roll requested 2024-12-07T00:51:43,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703109 2024-12-07T00:51:43,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703109 2024-12-07T00:51:43,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703109) roll requested 2024-12-07T00:51:43,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703115 2024-12-07T00:51:43,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703115 2024-12-07T00:51:43,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703115) roll requested 2024-12-07T00:51:43,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703121 2024-12-07T00:51:43,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703121 2024-12-07T00:51:43,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703121) roll requested 2024-12-07T00:51:43,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703127 2024-12-07T00:51:43,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703127 2024-12-07T00:51:43,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703127) roll requested 2024-12-07T00:51:43,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703133 2024-12-07T00:51:43,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703133 2024-12-07T00:51:43,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703133) roll requested 2024-12-07T00:51:43,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703139 2024-12-07T00:51:43,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703139 2024-12-07T00:51:43,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703139) roll requested 2024-12-07T00:51:43,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703145 2024-12-07T00:51:43,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703145 2024-12-07T00:51:43,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703145) roll requested 2024-12-07T00:51:43,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703151 2024-12-07T00:51:43,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703151 2024-12-07T00:51:43,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703151) roll requested 2024-12-07T00:51:43,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703159 2024-12-07T00:51:43,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703159 2024-12-07T00:51:43,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703159) roll requested 2024-12-07T00:51:43,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703165 2024-12-07T00:51:43,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703165 2024-12-07T00:51:43,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703165) roll requested 2024-12-07T00:51:43,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703172 2024-12-07T00:51:43,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703172 2024-12-07T00:51:43,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703172) roll requested 2024-12-07T00:51:43,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703178 2024-12-07T00:51:43,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703178 2024-12-07T00:51:43,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703178) roll requested 2024-12-07T00:51:43,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703184 2024-12-07T00:51:43,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703184 2024-12-07T00:51:43,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703184) roll requested 2024-12-07T00:51:43,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703192 2024-12-07T00:51:43,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703192 2024-12-07T00:51:43,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703192) roll requested 2024-12-07T00:51:43,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703198 2024-12-07T00:51:43,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703198 2024-12-07T00:51:43,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703198) roll requested 2024-12-07T00:51:43,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703208 2024-12-07T00:51:43,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703208 2024-12-07T00:51:43,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703208) roll requested 2024-12-07T00:51:43,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703214 2024-12-07T00:51:43,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703214 2024-12-07T00:51:43,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703214) roll requested 2024-12-07T00:51:43,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703222 2024-12-07T00:51:43,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703222 2024-12-07T00:51:43,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703222) roll requested 2024-12-07T00:51:43,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703227 2024-12-07T00:51:43,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703227 2024-12-07T00:51:43,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703227) roll requested 2024-12-07T00:51:43,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703234 2024-12-07T00:51:43,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703234 2024-12-07T00:51:43,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703234) roll requested 2024-12-07T00:51:43,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703240 2024-12-07T00:51:43,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703240 2024-12-07T00:51:43,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703240) roll requested 2024-12-07T00:51:43,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703248 2024-12-07T00:51:43,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703248 2024-12-07T00:51:43,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703248) roll requested 2024-12-07T00:51:43,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703254 2024-12-07T00:51:43,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703254 2024-12-07T00:51:43,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703254) roll requested 2024-12-07T00:51:43,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703260 2024-12-07T00:51:43,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703260 2024-12-07T00:51:43,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703260) roll requested 2024-12-07T00:51:43,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703267 2024-12-07T00:51:43,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703267 2024-12-07T00:51:43,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703267) roll requested 2024-12-07T00:51:43,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703273 2024-12-07T00:51:43,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703273 2024-12-07T00:51:43,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703273) roll requested 2024-12-07T00:51:43,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703279 2024-12-07T00:51:43,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703279 2024-12-07T00:51:43,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703279) roll requested 2024-12-07T00:51:43,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703285 2024-12-07T00:51:43,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703285 2024-12-07T00:51:43,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703285) roll requested 2024-12-07T00:51:43,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703291 2024-12-07T00:51:43,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703291 2024-12-07T00:51:43,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703291) roll requested 2024-12-07T00:51:43,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703297 2024-12-07T00:51:43,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703297 2024-12-07T00:51:43,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703297) roll requested 2024-12-07T00:51:43,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703304 2024-12-07T00:51:43,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703304 2024-12-07T00:51:43,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703304) roll requested 2024-12-07T00:51:43,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703311 2024-12-07T00:51:43,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703311 2024-12-07T00:51:43,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703304 is not closed yet, will try archiving it next time 2024-12-07T00:51:43,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703311) roll requested 2024-12-07T00:51:43,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703318 2024-12-07T00:51:43,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703318 2024-12-07T00:51:43,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703318) roll requested 2024-12-07T00:51:43,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703375 2024-12-07T00:51:43,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703375 2024-12-07T00:51:43,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703375) roll requested 2024-12-07T00:51:43,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703381 2024-12-07T00:51:43,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703381 2024-12-07T00:51:43,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703381) roll requested 2024-12-07T00:51:43,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703388 2024-12-07T00:51:43,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703388 2024-12-07T00:51:43,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703388) roll requested 2024-12-07T00:51:43,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703394 2024-12-07T00:51:43,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703394 2024-12-07T00:51:43,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703394) roll requested 2024-12-07T00:51:43,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703400 2024-12-07T00:51:43,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703400 2024-12-07T00:51:43,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703400) roll requested 2024-12-07T00:51:43,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703406 2024-12-07T00:51:43,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703406 2024-12-07T00:51:43,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703406) roll requested 2024-12-07T00:51:43,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703412 2024-12-07T00:51:43,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703412 2024-12-07T00:51:43,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703412) roll requested 2024-12-07T00:51:43,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703418 2024-12-07T00:51:43,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703418 2024-12-07T00:51:43,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703418) roll requested 2024-12-07T00:51:43,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703424 2024-12-07T00:51:43,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703424 2024-12-07T00:51:43,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703424) roll requested 2024-12-07T00:51:43,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703430 2024-12-07T00:51:43,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703430 2024-12-07T00:51:43,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703430) roll requested 2024-12-07T00:51:43,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703436 2024-12-07T00:51:43,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703436 2024-12-07T00:51:43,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703436) roll requested 2024-12-07T00:51:43,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703443 2024-12-07T00:51:43,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703443 2024-12-07T00:51:43,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703443) roll requested 2024-12-07T00:51:43,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703449 2024-12-07T00:51:43,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703449 2024-12-07T00:51:43,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703449) roll requested 2024-12-07T00:51:43,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703455 2024-12-07T00:51:43,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703455 2024-12-07T00:51:43,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703455) roll requested 2024-12-07T00:51:43,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703461 2024-12-07T00:51:43,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703461 2024-12-07T00:51:43,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703461) roll requested 2024-12-07T00:51:43,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703467 2024-12-07T00:51:43,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703467 2024-12-07T00:51:43,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703467) roll requested 2024-12-07T00:51:43,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703473 2024-12-07T00:51:43,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703473 2024-12-07T00:51:43,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703473) roll requested 2024-12-07T00:51:43,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703478 2024-12-07T00:51:43,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703478 2024-12-07T00:51:43,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703478) roll requested 2024-12-07T00:51:43,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703484 2024-12-07T00:51:43,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703484 2024-12-07T00:51:43,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703484) roll requested 2024-12-07T00:51:43,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703489 2024-12-07T00:51:43,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703489 2024-12-07T00:51:43,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703489) roll requested 2024-12-07T00:51:43,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703496 2024-12-07T00:51:43,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703496 2024-12-07T00:51:43,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703496) roll requested 2024-12-07T00:51:43,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703501 2024-12-07T00:51:43,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703501 2024-12-07T00:51:43,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703501) roll requested 2024-12-07T00:51:43,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703507 2024-12-07T00:51:43,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703507 2024-12-07T00:51:43,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703507) roll requested 2024-12-07T00:51:43,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703513 2024-12-07T00:51:43,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703513 2024-12-07T00:51:43,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703513) roll requested 2024-12-07T00:51:43,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703519 2024-12-07T00:51:43,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703519 2024-12-07T00:51:43,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703519) roll requested 2024-12-07T00:51:43,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703524 2024-12-07T00:51:43,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703524 2024-12-07T00:51:43,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703524) roll requested 2024-12-07T00:51:43,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703530 2024-12-07T00:51:43,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703530 2024-12-07T00:51:43,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703530) roll requested 2024-12-07T00:51:43,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703535 2024-12-07T00:51:43,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703535 2024-12-07T00:51:43,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703535) roll requested 2024-12-07T00:51:43,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703542 2024-12-07T00:51:43,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703542 2024-12-07T00:51:43,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703542) roll requested 2024-12-07T00:51:43,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703548 2024-12-07T00:51:43,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703548 2024-12-07T00:51:43,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703548) roll requested 2024-12-07T00:51:43,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703554 2024-12-07T00:51:43,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703554 2024-12-07T00:51:43,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703554) roll requested 2024-12-07T00:51:43,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703569 2024-12-07T00:51:43,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703569 2024-12-07T00:51:43,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703569) roll requested 2024-12-07T00:51:43,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703575 2024-12-07T00:51:43,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703575 2024-12-07T00:51:43,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703575) roll requested 2024-12-07T00:51:43,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703581 2024-12-07T00:51:43,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703581 2024-12-07T00:51:43,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703581) roll requested 2024-12-07T00:51:43,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703587 2024-12-07T00:51:43,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703587 2024-12-07T00:51:43,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703587) roll requested 2024-12-07T00:51:43,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703593 2024-12-07T00:51:43,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703593 2024-12-07T00:51:43,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703593) roll requested 2024-12-07T00:51:43,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703599 2024-12-07T00:51:43,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703599 2024-12-07T00:51:43,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703599) roll requested 2024-12-07T00:51:43,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703605 2024-12-07T00:51:43,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703605 2024-12-07T00:51:43,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703605) roll requested 2024-12-07T00:51:43,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703611 2024-12-07T00:51:43,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703611 2024-12-07T00:51:43,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703611) roll requested 2024-12-07T00:51:43,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703617 2024-12-07T00:51:43,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703617 2024-12-07T00:51:43,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703617) roll requested 2024-12-07T00:51:43,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703623 2024-12-07T00:51:43,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703623 2024-12-07T00:51:43,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703623) roll requested 2024-12-07T00:51:43,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703629 2024-12-07T00:51:43,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703629 2024-12-07T00:51:43,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703629) roll requested 2024-12-07T00:51:43,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703636 2024-12-07T00:51:43,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703636 2024-12-07T00:51:43,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703636) roll requested 2024-12-07T00:51:43,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703642 2024-12-07T00:51:43,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703642 2024-12-07T00:51:43,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703642) roll requested 2024-12-07T00:51:43,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703648 2024-12-07T00:51:43,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703648 2024-12-07T00:51:43,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703648) roll requested 2024-12-07T00:51:43,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703661 2024-12-07T00:51:43,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703661 2024-12-07T00:51:43,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703661) roll requested 2024-12-07T00:51:43,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703667 2024-12-07T00:51:43,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703667 2024-12-07T00:51:43,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703667) roll requested 2024-12-07T00:51:43,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703673 2024-12-07T00:51:43,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703673 2024-12-07T00:51:43,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703673) roll requested 2024-12-07T00:51:43,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703679 2024-12-07T00:51:43,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703679 2024-12-07T00:51:43,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703679) roll requested 2024-12-07T00:51:43,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703685 2024-12-07T00:51:43,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703685 2024-12-07T00:51:43,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703685) roll requested 2024-12-07T00:51:43,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703691 2024-12-07T00:51:43,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703691 2024-12-07T00:51:43,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703691) roll requested 2024-12-07T00:51:43,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703697 2024-12-07T00:51:43,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703697 2024-12-07T00:51:43,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703697) roll requested 2024-12-07T00:51:43,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703703 2024-12-07T00:51:43,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703703 2024-12-07T00:51:43,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703703) roll requested 2024-12-07T00:51:43,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703709 2024-12-07T00:51:43,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703709 2024-12-07T00:51:43,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703709) roll requested 2024-12-07T00:51:43,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703715 2024-12-07T00:51:43,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703715 2024-12-07T00:51:43,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703715) roll requested 2024-12-07T00:51:43,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703721 2024-12-07T00:51:43,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703721 2024-12-07T00:51:43,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703721) roll requested 2024-12-07T00:51:43,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703729 2024-12-07T00:51:43,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703729 2024-12-07T00:51:43,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703729) roll requested 2024-12-07T00:51:43,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703737 2024-12-07T00:51:43,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703737 2024-12-07T00:51:43,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703737) roll requested 2024-12-07T00:51:43,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703743 2024-12-07T00:51:43,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703743 2024-12-07T00:51:43,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703743) roll requested 2024-12-07T00:51:43,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703748 2024-12-07T00:51:43,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703748 2024-12-07T00:51:43,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703748) roll requested 2024-12-07T00:51:43,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703754 2024-12-07T00:51:43,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703754 2024-12-07T00:51:43,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703754) roll requested 2024-12-07T00:51:43,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703761 2024-12-07T00:51:43,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703761 2024-12-07T00:51:43,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703761) roll requested 2024-12-07T00:51:43,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703767 2024-12-07T00:51:43,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703767 2024-12-07T00:51:43,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703767) roll requested 2024-12-07T00:51:43,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703773 2024-12-07T00:51:43,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703773 2024-12-07T00:51:43,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703773) roll requested 2024-12-07T00:51:43,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703779 2024-12-07T00:51:43,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703779 2024-12-07T00:51:43,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703779) roll requested 2024-12-07T00:51:43,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703785 2024-12-07T00:51:43,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703785 2024-12-07T00:51:43,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703785) roll requested 2024-12-07T00:51:43,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703791 2024-12-07T00:51:43,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703791 2024-12-07T00:51:43,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703791) roll requested 2024-12-07T00:51:43,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703796 2024-12-07T00:51:43,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703796 2024-12-07T00:51:43,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703796) roll requested 2024-12-07T00:51:43,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703802 2024-12-07T00:51:43,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703802 2024-12-07T00:51:43,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703802) roll requested 2024-12-07T00:51:43,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703807 2024-12-07T00:51:43,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703807 2024-12-07T00:51:43,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703807) roll requested 2024-12-07T00:51:43,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703813 2024-12-07T00:51:43,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703813 2024-12-07T00:51:43,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703813) roll requested 2024-12-07T00:51:43,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703819 2024-12-07T00:51:43,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703819 2024-12-07T00:51:43,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703819) roll requested 2024-12-07T00:51:43,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703824 2024-12-07T00:51:43,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703824 2024-12-07T00:51:43,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703824) roll requested 2024-12-07T00:51:43,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703830 2024-12-07T00:51:43,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703830 2024-12-07T00:51:43,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703830) roll requested 2024-12-07T00:51:43,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703836 2024-12-07T00:51:43,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703836 2024-12-07T00:51:43,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703836) roll requested 2024-12-07T00:51:43,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703842 2024-12-07T00:51:43,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703842 2024-12-07T00:51:43,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703842) roll requested 2024-12-07T00:51:43,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703847 2024-12-07T00:51:43,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703847 2024-12-07T00:51:43,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703847) roll requested 2024-12-07T00:51:43,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703853 2024-12-07T00:51:43,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703853 2024-12-07T00:51:43,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703853) roll requested 2024-12-07T00:51:43,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703859 2024-12-07T00:51:43,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703859 2024-12-07T00:51:43,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703859) roll requested 2024-12-07T00:51:43,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703865 2024-12-07T00:51:43,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703865 2024-12-07T00:51:43,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703865) roll requested 2024-12-07T00:51:43,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703870 2024-12-07T00:51:43,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703870 2024-12-07T00:51:43,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703870) roll requested 2024-12-07T00:51:43,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703876 2024-12-07T00:51:43,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703876 2024-12-07T00:51:43,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703876) roll requested 2024-12-07T00:51:43,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703882 2024-12-07T00:51:43,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703882 2024-12-07T00:51:43,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703882) roll requested 2024-12-07T00:51:43,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703887 2024-12-07T00:51:43,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703887 2024-12-07T00:51:43,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703887) roll requested 2024-12-07T00:51:43,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703893 2024-12-07T00:51:43,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703893 2024-12-07T00:51:43,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703893) roll requested 2024-12-07T00:51:43,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703899 2024-12-07T00:51:43,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703899 2024-12-07T00:51:43,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703899) roll requested 2024-12-07T00:51:43,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703909 2024-12-07T00:51:43,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703909 2024-12-07T00:51:43,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703909) roll requested 2024-12-07T00:51:43,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703918 2024-12-07T00:51:43,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703918 2024-12-07T00:51:43,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703918) roll requested 2024-12-07T00:51:43,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703923 2024-12-07T00:51:43,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703923 2024-12-07T00:51:43,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703923) roll requested 2024-12-07T00:51:43,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703929 2024-12-07T00:51:43,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703929 2024-12-07T00:51:43,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703929) roll requested 2024-12-07T00:51:43,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703935 2024-12-07T00:51:43,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703935 2024-12-07T00:51:43,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703935) roll requested 2024-12-07T00:51:43,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703941 2024-12-07T00:51:43,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703941 2024-12-07T00:51:43,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703941) roll requested 2024-12-07T00:51:43,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703947 2024-12-07T00:51:43,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703947 2024-12-07T00:51:43,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703947) roll requested 2024-12-07T00:51:43,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703953 2024-12-07T00:51:43,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703953 2024-12-07T00:51:43,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703953) roll requested 2024-12-07T00:51:43,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703964 2024-12-07T00:51:43,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703964 2024-12-07T00:51:43,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703964) roll requested 2024-12-07T00:51:43,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703971 2024-12-07T00:51:43,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703971 2024-12-07T00:51:43,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703971) roll requested 2024-12-07T00:51:43,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703976 2024-12-07T00:51:43,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703976 2024-12-07T00:51:43,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703976) roll requested 2024-12-07T00:51:43,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703983 2024-12-07T00:51:43,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703983 2024-12-07T00:51:43,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703983) roll requested 2024-12-07T00:51:43,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703989 2024-12-07T00:51:43,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:43,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703989 2024-12-07T00:51:43,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:43,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:43,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:43,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:43,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703989) roll requested 2024-12-07T00:51:43,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532703996 2024-12-07T00:51:44,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703996 2024-12-07T00:51:44,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532703996) roll requested 2024-12-07T00:51:44,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704002 2024-12-07T00:51:44,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532703996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704002 2024-12-07T00:51:44,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704002) roll requested 2024-12-07T00:51:44,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704007 2024-12-07T00:51:44,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704007 2024-12-07T00:51:44,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704007) roll requested 2024-12-07T00:51:44,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704013 2024-12-07T00:51:44,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704013 2024-12-07T00:51:44,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704013) roll requested 2024-12-07T00:51:44,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704019 2024-12-07T00:51:44,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704019 2024-12-07T00:51:44,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704019) roll requested 2024-12-07T00:51:44,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704024 2024-12-07T00:51:44,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704024 2024-12-07T00:51:44,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704024) roll requested 2024-12-07T00:51:44,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704031 2024-12-07T00:51:44,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704031 2024-12-07T00:51:44,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704031) roll requested 2024-12-07T00:51:44,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704036 2024-12-07T00:51:44,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704036 2024-12-07T00:51:44,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704036) roll requested 2024-12-07T00:51:44,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704042 2024-12-07T00:51:44,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704042 2024-12-07T00:51:44,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704042) roll requested 2024-12-07T00:51:44,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704048 2024-12-07T00:51:44,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704048 2024-12-07T00:51:44,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704048) roll requested 2024-12-07T00:51:44,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704054 2024-12-07T00:51:44,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704054 2024-12-07T00:51:44,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704054) roll requested 2024-12-07T00:51:44,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704062 2024-12-07T00:51:44,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704062 2024-12-07T00:51:44,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704062) roll requested 2024-12-07T00:51:44,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704068 2024-12-07T00:51:44,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704068 2024-12-07T00:51:44,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704068) roll requested 2024-12-07T00:51:44,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704074 2024-12-07T00:51:44,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704074 2024-12-07T00:51:44,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704074) roll requested 2024-12-07T00:51:44,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704079 2024-12-07T00:51:44,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704079 2024-12-07T00:51:44,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704079) roll requested 2024-12-07T00:51:44,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704085 2024-12-07T00:51:44,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704085 2024-12-07T00:51:44,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704085) roll requested 2024-12-07T00:51:44,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704091 2024-12-07T00:51:44,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704091 2024-12-07T00:51:44,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704091) roll requested 2024-12-07T00:51:44,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704096 2024-12-07T00:51:44,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704096 2024-12-07T00:51:44,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704096) roll requested 2024-12-07T00:51:44,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704102 2024-12-07T00:51:44,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704102 2024-12-07T00:51:44,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704102) roll requested 2024-12-07T00:51:44,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704108 2024-12-07T00:51:44,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704108 2024-12-07T00:51:44,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704108) roll requested 2024-12-07T00:51:44,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704113 2024-12-07T00:51:44,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704113 2024-12-07T00:51:44,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704113) roll requested 2024-12-07T00:51:44,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704120 2024-12-07T00:51:44,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704120 2024-12-07T00:51:44,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704120) roll requested 2024-12-07T00:51:44,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704126 2024-12-07T00:51:44,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704126 2024-12-07T00:51:44,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704126) roll requested 2024-12-07T00:51:44,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704131 2024-12-07T00:51:44,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704131 2024-12-07T00:51:44,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704131) roll requested 2024-12-07T00:51:44,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704137 2024-12-07T00:51:44,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704137 2024-12-07T00:51:44,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704137) roll requested 2024-12-07T00:51:44,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704143 2024-12-07T00:51:44,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704143 2024-12-07T00:51:44,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704143) roll requested 2024-12-07T00:51:44,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704148 2024-12-07T00:51:44,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704148 2024-12-07T00:51:44,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704148) roll requested 2024-12-07T00:51:44,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704154 2024-12-07T00:51:44,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704154 2024-12-07T00:51:44,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704154) roll requested 2024-12-07T00:51:44,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704160 2024-12-07T00:51:44,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704160 2024-12-07T00:51:44,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704160) roll requested 2024-12-07T00:51:44,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704167 2024-12-07T00:51:44,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704167 2024-12-07T00:51:44,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704167) roll requested 2024-12-07T00:51:44,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704172 2024-12-07T00:51:44,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704172 2024-12-07T00:51:44,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704172) roll requested 2024-12-07T00:51:44,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704178 2024-12-07T00:51:44,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704178 2024-12-07T00:51:44,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704178) roll requested 2024-12-07T00:51:44,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704184 2024-12-07T00:51:44,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704184 2024-12-07T00:51:44,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704184) roll requested 2024-12-07T00:51:44,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704189 2024-12-07T00:51:44,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704189 2024-12-07T00:51:44,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704189) roll requested 2024-12-07T00:51:44,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704195 2024-12-07T00:51:44,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704195 2024-12-07T00:51:44,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704195) roll requested 2024-12-07T00:51:44,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704200 2024-12-07T00:51:44,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704200 2024-12-07T00:51:44,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704200) roll requested 2024-12-07T00:51:44,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704206 2024-12-07T00:51:44,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704206 2024-12-07T00:51:44,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704206) roll requested 2024-12-07T00:51:44,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704212 2024-12-07T00:51:44,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704212 2024-12-07T00:51:44,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704212) roll requested 2024-12-07T00:51:44,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704218 2024-12-07T00:51:44,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704218 2024-12-07T00:51:44,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704218) roll requested 2024-12-07T00:51:44,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704223 2024-12-07T00:51:44,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704223 2024-12-07T00:51:44,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704223) roll requested 2024-12-07T00:51:44,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704229 2024-12-07T00:51:44,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704229 2024-12-07T00:51:44,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704229) roll requested 2024-12-07T00:51:44,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704235 2024-12-07T00:51:44,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704235 2024-12-07T00:51:44,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704235) roll requested 2024-12-07T00:51:44,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704240 2024-12-07T00:51:44,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704240 2024-12-07T00:51:44,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704240) roll requested 2024-12-07T00:51:44,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704247 2024-12-07T00:51:44,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704247 2024-12-07T00:51:44,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704247) roll requested 2024-12-07T00:51:44,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704252 2024-12-07T00:51:44,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704252 2024-12-07T00:51:44,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704252) roll requested 2024-12-07T00:51:44,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704259 2024-12-07T00:51:44,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704259 2024-12-07T00:51:44,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704259) roll requested 2024-12-07T00:51:44,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704265 2024-12-07T00:51:44,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704265 2024-12-07T00:51:44,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704265) roll requested 2024-12-07T00:51:44,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704271 2024-12-07T00:51:44,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704271 2024-12-07T00:51:44,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704271) roll requested 2024-12-07T00:51:44,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704276 2024-12-07T00:51:44,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704276 2024-12-07T00:51:44,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704276) roll requested 2024-12-07T00:51:44,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704282 2024-12-07T00:51:44,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704282 2024-12-07T00:51:44,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704282) roll requested 2024-12-07T00:51:44,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704288 2024-12-07T00:51:44,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704288 2024-12-07T00:51:44,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704288) roll requested 2024-12-07T00:51:44,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704294 2024-12-07T00:51:44,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704294 2024-12-07T00:51:44,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704294) roll requested 2024-12-07T00:51:44,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704309 2024-12-07T00:51:44,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704309 2024-12-07T00:51:44,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704309) roll requested 2024-12-07T00:51:44,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704317 2024-12-07T00:51:44,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704317 2024-12-07T00:51:44,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704317) roll requested 2024-12-07T00:51:44,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704327 2024-12-07T00:51:44,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704327 2024-12-07T00:51:44,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704327) roll requested 2024-12-07T00:51:44,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704334 2024-12-07T00:51:44,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704334 2024-12-07T00:51:44,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704334) roll requested 2024-12-07T00:51:44,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704341 2024-12-07T00:51:44,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704341 2024-12-07T00:51:44,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704341) roll requested 2024-12-07T00:51:44,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704348 2024-12-07T00:51:44,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704348 2024-12-07T00:51:44,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704348) roll requested 2024-12-07T00:51:44,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704355 2024-12-07T00:51:44,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704355 2024-12-07T00:51:44,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704355) roll requested 2024-12-07T00:51:44,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704363 2024-12-07T00:51:44,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704363 2024-12-07T00:51:44,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704363) roll requested 2024-12-07T00:51:44,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704368 2024-12-07T00:51:44,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704368 2024-12-07T00:51:44,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704368) roll requested 2024-12-07T00:51:44,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704374 2024-12-07T00:51:44,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704374 2024-12-07T00:51:44,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704374) roll requested 2024-12-07T00:51:44,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704379 2024-12-07T00:51:44,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704379 2024-12-07T00:51:44,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704379) roll requested 2024-12-07T00:51:44,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704385 2024-12-07T00:51:44,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704385 2024-12-07T00:51:44,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704385) roll requested 2024-12-07T00:51:44,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704390 2024-12-07T00:51:44,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704390 2024-12-07T00:51:44,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704390) roll requested 2024-12-07T00:51:44,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704396 2024-12-07T00:51:44,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704396 2024-12-07T00:51:44,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704396) roll requested 2024-12-07T00:51:44,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704401 2024-12-07T00:51:44,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704401 2024-12-07T00:51:44,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704401) roll requested 2024-12-07T00:51:44,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704407 2024-12-07T00:51:44,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704407 2024-12-07T00:51:44,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704407) roll requested 2024-12-07T00:51:44,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704412 2024-12-07T00:51:44,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704412 2024-12-07T00:51:44,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704412) roll requested 2024-12-07T00:51:44,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704417 2024-12-07T00:51:44,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704417 2024-12-07T00:51:44,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704417) roll requested 2024-12-07T00:51:44,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704423 2024-12-07T00:51:44,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704423 2024-12-07T00:51:44,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704423) roll requested 2024-12-07T00:51:44,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704428 2024-12-07T00:51:44,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704428 2024-12-07T00:51:44,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704428) roll requested 2024-12-07T00:51:44,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704433 2024-12-07T00:51:44,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704433 2024-12-07T00:51:44,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704433) roll requested 2024-12-07T00:51:44,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704439 2024-12-07T00:51:44,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704439 2024-12-07T00:51:44,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704439) roll requested 2024-12-07T00:51:44,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704445 2024-12-07T00:51:44,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704445 2024-12-07T00:51:44,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704445) roll requested 2024-12-07T00:51:44,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704450 2024-12-07T00:51:44,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704450 2024-12-07T00:51:44,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704450) roll requested 2024-12-07T00:51:44,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704456 2024-12-07T00:51:44,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704456 2024-12-07T00:51:44,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704456) roll requested 2024-12-07T00:51:44,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704462 2024-12-07T00:51:44,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704462 2024-12-07T00:51:44,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704462) roll requested 2024-12-07T00:51:44,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704468 2024-12-07T00:51:44,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704468 2024-12-07T00:51:44,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704468) roll requested 2024-12-07T00:51:44,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704474 2024-12-07T00:51:44,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704474 2024-12-07T00:51:44,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704474) roll requested 2024-12-07T00:51:44,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704479 2024-12-07T00:51:44,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704479 2024-12-07T00:51:44,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704479) roll requested 2024-12-07T00:51:44,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704485 2024-12-07T00:51:44,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704485 2024-12-07T00:51:44,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704485) roll requested 2024-12-07T00:51:44,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704490 2024-12-07T00:51:44,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704490 2024-12-07T00:51:44,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704490) roll requested 2024-12-07T00:51:44,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704497 2024-12-07T00:51:44,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704497 2024-12-07T00:51:44,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704497) roll requested 2024-12-07T00:51:44,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704502 2024-12-07T00:51:44,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704502 2024-12-07T00:51:44,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704502) roll requested 2024-12-07T00:51:44,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704509 2024-12-07T00:51:44,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704509 2024-12-07T00:51:44,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704509) roll requested 2024-12-07T00:51:44,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704515 2024-12-07T00:51:44,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704515 2024-12-07T00:51:44,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704515) roll requested 2024-12-07T00:51:44,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704520 2024-12-07T00:51:44,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704520 2024-12-07T00:51:44,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704520) roll requested 2024-12-07T00:51:44,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704526 2024-12-07T00:51:44,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704526 2024-12-07T00:51:44,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704526) roll requested 2024-12-07T00:51:44,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704532 2024-12-07T00:51:44,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704532 2024-12-07T00:51:44,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704532) roll requested 2024-12-07T00:51:44,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704537 2024-12-07T00:51:44,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704537 2024-12-07T00:51:44,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704537) roll requested 2024-12-07T00:51:44,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704543 2024-12-07T00:51:44,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704543 2024-12-07T00:51:44,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704543) roll requested 2024-12-07T00:51:44,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704549 2024-12-07T00:51:44,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704549 2024-12-07T00:51:44,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704549) roll requested 2024-12-07T00:51:44,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704555 2024-12-07T00:51:44,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704555 2024-12-07T00:51:44,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704555) roll requested 2024-12-07T00:51:44,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704561 2024-12-07T00:51:44,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704561 2024-12-07T00:51:44,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704561) roll requested 2024-12-07T00:51:44,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704567 2024-12-07T00:51:44,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704567 2024-12-07T00:51:44,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704567) roll requested 2024-12-07T00:51:44,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704573 2024-12-07T00:51:44,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704573 2024-12-07T00:51:44,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704573) roll requested 2024-12-07T00:51:44,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704578 2024-12-07T00:51:44,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704578 2024-12-07T00:51:44,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704578) roll requested 2024-12-07T00:51:44,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704584 2024-12-07T00:51:44,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704584 2024-12-07T00:51:44,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704584) roll requested 2024-12-07T00:51:44,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704590 2024-12-07T00:51:44,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704590 2024-12-07T00:51:44,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704590) roll requested 2024-12-07T00:51:44,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704595 2024-12-07T00:51:44,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704595 2024-12-07T00:51:44,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704595) roll requested 2024-12-07T00:51:44,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704601 2024-12-07T00:51:44,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704601 2024-12-07T00:51:44,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704601) roll requested 2024-12-07T00:51:44,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704607 2024-12-07T00:51:44,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704607 2024-12-07T00:51:44,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704607) roll requested 2024-12-07T00:51:44,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704613 2024-12-07T00:51:44,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704613 2024-12-07T00:51:44,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704613) roll requested 2024-12-07T00:51:44,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704618 2024-12-07T00:51:44,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704618 2024-12-07T00:51:44,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704618) roll requested 2024-12-07T00:51:44,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704624 2024-12-07T00:51:44,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704624 2024-12-07T00:51:44,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704624) roll requested 2024-12-07T00:51:44,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704629 2024-12-07T00:51:44,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704629 2024-12-07T00:51:44,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704629) roll requested 2024-12-07T00:51:44,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704635 2024-12-07T00:51:44,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704635 2024-12-07T00:51:44,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704635) roll requested 2024-12-07T00:51:44,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704641 2024-12-07T00:51:44,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704641 2024-12-07T00:51:44,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704641) roll requested 2024-12-07T00:51:44,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704647 2024-12-07T00:51:44,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704647 2024-12-07T00:51:44,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704647) roll requested 2024-12-07T00:51:44,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704653 2024-12-07T00:51:44,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704653 2024-12-07T00:51:44,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704653) roll requested 2024-12-07T00:51:44,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704659 2024-12-07T00:51:44,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704659 2024-12-07T00:51:44,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704659) roll requested 2024-12-07T00:51:44,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704665 2024-12-07T00:51:44,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704665 2024-12-07T00:51:44,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704665) roll requested 2024-12-07T00:51:44,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704670 2024-12-07T00:51:44,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704670 2024-12-07T00:51:44,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704670) roll requested 2024-12-07T00:51:44,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704676 2024-12-07T00:51:44,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704676 2024-12-07T00:51:44,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704676) roll requested 2024-12-07T00:51:44,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704682 2024-12-07T00:51:44,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704682 2024-12-07T00:51:44,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704682) roll requested 2024-12-07T00:51:44,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704688 2024-12-07T00:51:44,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704688 2024-12-07T00:51:44,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704688) roll requested 2024-12-07T00:51:44,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704694 2024-12-07T00:51:44,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704694 2024-12-07T00:51:44,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704694) roll requested 2024-12-07T00:51:44,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704699 2024-12-07T00:51:44,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704699 2024-12-07T00:51:44,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704699) roll requested 2024-12-07T00:51:44,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704706 2024-12-07T00:51:44,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704706 2024-12-07T00:51:44,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704706) roll requested 2024-12-07T00:51:44,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704712 2024-12-07T00:51:44,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704712 2024-12-07T00:51:44,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704712) roll requested 2024-12-07T00:51:44,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704718 2024-12-07T00:51:44,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704718 2024-12-07T00:51:44,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704718) roll requested 2024-12-07T00:51:44,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704724 2024-12-07T00:51:44,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704724 2024-12-07T00:51:44,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704724) roll requested 2024-12-07T00:51:44,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704729 2024-12-07T00:51:44,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704729 2024-12-07T00:51:44,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704729) roll requested 2024-12-07T00:51:44,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704735 2024-12-07T00:51:44,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704735 2024-12-07T00:51:44,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704735) roll requested 2024-12-07T00:51:44,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704741 2024-12-07T00:51:44,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704741 2024-12-07T00:51:44,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704741) roll requested 2024-12-07T00:51:44,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704747 2024-12-07T00:51:44,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704747 2024-12-07T00:51:44,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704747) roll requested 2024-12-07T00:51:44,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704752 2024-12-07T00:51:44,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704752 2024-12-07T00:51:44,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704752) roll requested 2024-12-07T00:51:44,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704758 2024-12-07T00:51:44,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704758 2024-12-07T00:51:44,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704758) roll requested 2024-12-07T00:51:44,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704764 2024-12-07T00:51:44,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704764 2024-12-07T00:51:44,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704764) roll requested 2024-12-07T00:51:44,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704770 2024-12-07T00:51:44,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704770 2024-12-07T00:51:44,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704770) roll requested 2024-12-07T00:51:44,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704776 2024-12-07T00:51:44,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704776 2024-12-07T00:51:44,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704776) roll requested 2024-12-07T00:51:44,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704782 2024-12-07T00:51:44,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704782 2024-12-07T00:51:44,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704782) roll requested 2024-12-07T00:51:44,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704787 2024-12-07T00:51:44,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704787 2024-12-07T00:51:44,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704787) roll requested 2024-12-07T00:51:44,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704793 2024-12-07T00:51:44,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704793 2024-12-07T00:51:44,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704793) roll requested 2024-12-07T00:51:44,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704799 2024-12-07T00:51:44,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704799 2024-12-07T00:51:44,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704799) roll requested 2024-12-07T00:51:44,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704805 2024-12-07T00:51:44,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704805 2024-12-07T00:51:44,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704805) roll requested 2024-12-07T00:51:44,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704810 2024-12-07T00:51:44,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704810 2024-12-07T00:51:44,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704810) roll requested 2024-12-07T00:51:44,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704815 2024-12-07T00:51:44,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704815 2024-12-07T00:51:44,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704815) roll requested 2024-12-07T00:51:44,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704821 2024-12-07T00:51:44,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704821 2024-12-07T00:51:44,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704821) roll requested 2024-12-07T00:51:44,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704827 2024-12-07T00:51:44,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704827 2024-12-07T00:51:44,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704827) roll requested 2024-12-07T00:51:44,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704833 2024-12-07T00:51:44,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704833 2024-12-07T00:51:44,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704833) roll requested 2024-12-07T00:51:44,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704839 2024-12-07T00:51:44,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704839 2024-12-07T00:51:44,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704839) roll requested 2024-12-07T00:51:44,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704845 2024-12-07T00:51:44,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704845 2024-12-07T00:51:44,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704845) roll requested 2024-12-07T00:51:44,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704851 2024-12-07T00:51:44,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704851 2024-12-07T00:51:44,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704851) roll requested 2024-12-07T00:51:44,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704856 2024-12-07T00:51:44,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704856 2024-12-07T00:51:44,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704856) roll requested 2024-12-07T00:51:44,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704862 2024-12-07T00:51:44,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704862 2024-12-07T00:51:44,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704862) roll requested 2024-12-07T00:51:44,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704868 2024-12-07T00:51:44,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704868 2024-12-07T00:51:44,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704868) roll requested 2024-12-07T00:51:44,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704874 2024-12-07T00:51:44,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704874 2024-12-07T00:51:44,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704874) roll requested 2024-12-07T00:51:44,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704880 2024-12-07T00:51:44,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704880 2024-12-07T00:51:44,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704880) roll requested 2024-12-07T00:51:44,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704885 2024-12-07T00:51:44,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704885 2024-12-07T00:51:44,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704885) roll requested 2024-12-07T00:51:44,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704890 2024-12-07T00:51:44,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704890 2024-12-07T00:51:44,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704890) roll requested 2024-12-07T00:51:44,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704896 2024-12-07T00:51:44,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704896 2024-12-07T00:51:44,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704896) roll requested 2024-12-07T00:51:44,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704902 2024-12-07T00:51:44,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704902 2024-12-07T00:51:44,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704902) roll requested 2024-12-07T00:51:44,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704909 2024-12-07T00:51:44,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704909 2024-12-07T00:51:44,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704909) roll requested 2024-12-07T00:51:44,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704915 2024-12-07T00:51:44,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704915 2024-12-07T00:51:44,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704915) roll requested 2024-12-07T00:51:44,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704921 2024-12-07T00:51:44,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704921 2024-12-07T00:51:44,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704921) roll requested 2024-12-07T00:51:44,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704926 2024-12-07T00:51:44,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704926 2024-12-07T00:51:44,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704926) roll requested 2024-12-07T00:51:44,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704932 2024-12-07T00:51:44,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704932 2024-12-07T00:51:44,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704932) roll requested 2024-12-07T00:51:44,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704938 2024-12-07T00:51:44,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704938 2024-12-07T00:51:44,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704938) roll requested 2024-12-07T00:51:44,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704944 2024-12-07T00:51:44,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704944 2024-12-07T00:51:44,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704944) roll requested 2024-12-07T00:51:44,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704950 2024-12-07T00:51:44,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704950 2024-12-07T00:51:44,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704950) roll requested 2024-12-07T00:51:44,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704956 2024-12-07T00:51:44,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704956 2024-12-07T00:51:44,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704956) roll requested 2024-12-07T00:51:44,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704961 2024-12-07T00:51:44,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704961 2024-12-07T00:51:44,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704961) roll requested 2024-12-07T00:51:44,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704967 2024-12-07T00:51:44,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704967 2024-12-07T00:51:44,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704967) roll requested 2024-12-07T00:51:44,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704973 2024-12-07T00:51:44,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704973 2024-12-07T00:51:44,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704973) roll requested 2024-12-07T00:51:44,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704979 2024-12-07T00:51:44,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704979 2024-12-07T00:51:44,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704979) roll requested 2024-12-07T00:51:44,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704984 2024-12-07T00:51:44,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704984 2024-12-07T00:51:44,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704984) roll requested 2024-12-07T00:51:44,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704990 2024-12-07T00:51:44,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:44,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704990 2024-12-07T00:51:44,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:44,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:44,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:44,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:44,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704990) roll requested 2024-12-07T00:51:44,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532704997 2024-12-07T00:51:45,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704997 2024-12-07T00:51:45,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532704997) roll requested 2024-12-07T00:51:45,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705003 2024-12-07T00:51:45,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532704997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705003 2024-12-07T00:51:45,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705003) roll requested 2024-12-07T00:51:45,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705009 2024-12-07T00:51:45,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705009 2024-12-07T00:51:45,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705009) roll requested 2024-12-07T00:51:45,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705015 2024-12-07T00:51:45,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705015 2024-12-07T00:51:45,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705015) roll requested 2024-12-07T00:51:45,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705020 2024-12-07T00:51:45,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705020 2024-12-07T00:51:45,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1282, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705020) roll requested 2024-12-07T00:51:45,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705026 2024-12-07T00:51:45,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705026 2024-12-07T00:51:45,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1283, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705026) roll requested 2024-12-07T00:51:45,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705032 2024-12-07T00:51:45,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705032 2024-12-07T00:51:45,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1284, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705032) roll requested 2024-12-07T00:51:45,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705037 2024-12-07T00:51:45,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705037 2024-12-07T00:51:45,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1285, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705037) roll requested 2024-12-07T00:51:45,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705044 2024-12-07T00:51:45,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705044 2024-12-07T00:51:45,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1286, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705044) roll requested 2024-12-07T00:51:45,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705050 2024-12-07T00:51:45,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705050 2024-12-07T00:51:45,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1287, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705050) roll requested 2024-12-07T00:51:45,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705060 2024-12-07T00:51:45,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705060 2024-12-07T00:51:45,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1288, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705060) roll requested 2024-12-07T00:51:45,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705070 2024-12-07T00:51:45,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705070 2024-12-07T00:51:45,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1289, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705070) roll requested 2024-12-07T00:51:45,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705079 2024-12-07T00:51:45,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705079 2024-12-07T00:51:45,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1290, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705079) roll requested 2024-12-07T00:51:45,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705085 2024-12-07T00:51:45,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705085 2024-12-07T00:51:45,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1291, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705085) roll requested 2024-12-07T00:51:45,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705091 2024-12-07T00:51:45,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705091 2024-12-07T00:51:45,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1292, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705091) roll requested 2024-12-07T00:51:45,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705097 2024-12-07T00:51:45,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705097 2024-12-07T00:51:45,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1293, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705097) roll requested 2024-12-07T00:51:45,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705103 2024-12-07T00:51:45,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705103 2024-12-07T00:51:45,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1294, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705103) roll requested 2024-12-07T00:51:45,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705109 2024-12-07T00:51:45,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705109 2024-12-07T00:51:45,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1295, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705109) roll requested 2024-12-07T00:51:45,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705115 2024-12-07T00:51:45,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705115 2024-12-07T00:51:45,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1296, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705115) roll requested 2024-12-07T00:51:45,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705120 2024-12-07T00:51:45,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705120 2024-12-07T00:51:45,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1297, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705120) roll requested 2024-12-07T00:51:45,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705126 2024-12-07T00:51:45,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705126 2024-12-07T00:51:45,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1298, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705126) roll requested 2024-12-07T00:51:45,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705132 2024-12-07T00:51:45,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705132 2024-12-07T00:51:45,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1299, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705132) roll requested 2024-12-07T00:51:45,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705138 2024-12-07T00:51:45,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705138 2024-12-07T00:51:45,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1300, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705138) roll requested 2024-12-07T00:51:45,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705144 2024-12-07T00:51:45,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705144 2024-12-07T00:51:45,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1301, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705144) roll requested 2024-12-07T00:51:45,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705150 2024-12-07T00:51:45,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705150 2024-12-07T00:51:45,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1302, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705150) roll requested 2024-12-07T00:51:45,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705156 2024-12-07T00:51:45,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705156 2024-12-07T00:51:45,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1303, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705156) roll requested 2024-12-07T00:51:45,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705162 2024-12-07T00:51:45,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705162 2024-12-07T00:51:45,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1304, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705162) roll requested 2024-12-07T00:51:45,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705169 2024-12-07T00:51:45,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705169 2024-12-07T00:51:45,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1305, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705169) roll requested 2024-12-07T00:51:45,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705175 2024-12-07T00:51:45,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705175 2024-12-07T00:51:45,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1306, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705175) roll requested 2024-12-07T00:51:45,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705181 2024-12-07T00:51:45,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705181 2024-12-07T00:51:45,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1307, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705181) roll requested 2024-12-07T00:51:45,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705187 2024-12-07T00:51:45,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705187 2024-12-07T00:51:45,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1308, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705187) roll requested 2024-12-07T00:51:45,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705193 2024-12-07T00:51:45,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705193 2024-12-07T00:51:45,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1309, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705193) roll requested 2024-12-07T00:51:45,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705199 2024-12-07T00:51:45,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705199 2024-12-07T00:51:45,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1310, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705199) roll requested 2024-12-07T00:51:45,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705205 2024-12-07T00:51:45,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705205 2024-12-07T00:51:45,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1311, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705205) roll requested 2024-12-07T00:51:45,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705211 2024-12-07T00:51:45,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705211 2024-12-07T00:51:45,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1312, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705211) roll requested 2024-12-07T00:51:45,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705217 2024-12-07T00:51:45,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705217 2024-12-07T00:51:45,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1313, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705217) roll requested 2024-12-07T00:51:45,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705223 2024-12-07T00:51:45,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705223 2024-12-07T00:51:45,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1314, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705223) roll requested 2024-12-07T00:51:45,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705229 2024-12-07T00:51:45,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705229 2024-12-07T00:51:45,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1315, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705229) roll requested 2024-12-07T00:51:45,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705235 2024-12-07T00:51:45,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705235 2024-12-07T00:51:45,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1316, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705235) roll requested 2024-12-07T00:51:45,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705241 2024-12-07T00:51:45,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705241 2024-12-07T00:51:45,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1317, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705241) roll requested 2024-12-07T00:51:45,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705247 2024-12-07T00:51:45,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705247 2024-12-07T00:51:45,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1318, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705247) roll requested 2024-12-07T00:51:45,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705253 2024-12-07T00:51:45,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705253 2024-12-07T00:51:45,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1319, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705253) roll requested 2024-12-07T00:51:45,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705259 2024-12-07T00:51:45,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705259 2024-12-07T00:51:45,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1320, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705259) roll requested 2024-12-07T00:51:45,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705265 2024-12-07T00:51:45,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705265 2024-12-07T00:51:45,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1321, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705265) roll requested 2024-12-07T00:51:45,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705271 2024-12-07T00:51:45,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705271 2024-12-07T00:51:45,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1322, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705271) roll requested 2024-12-07T00:51:45,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705277 2024-12-07T00:51:45,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705277 2024-12-07T00:51:45,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1323, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705277) roll requested 2024-12-07T00:51:45,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705284 2024-12-07T00:51:45,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705284 2024-12-07T00:51:45,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1324, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705284) roll requested 2024-12-07T00:51:45,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705290 2024-12-07T00:51:45,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705290 2024-12-07T00:51:45,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1325, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705290) roll requested 2024-12-07T00:51:45,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705297 2024-12-07T00:51:45,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705297 2024-12-07T00:51:45,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1326, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705297) roll requested 2024-12-07T00:51:45,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705303 2024-12-07T00:51:45,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705303 2024-12-07T00:51:45,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1327, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705303) roll requested 2024-12-07T00:51:45,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705308 2024-12-07T00:51:45,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705308 2024-12-07T00:51:45,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1328, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705308) roll requested 2024-12-07T00:51:45,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705315 2024-12-07T00:51:45,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705315 2024-12-07T00:51:45,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1329, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705315) roll requested 2024-12-07T00:51:45,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705320 2024-12-07T00:51:45,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705320 2024-12-07T00:51:45,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1330, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705320) roll requested 2024-12-07T00:51:45,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705326 2024-12-07T00:51:45,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705326 2024-12-07T00:51:45,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1331, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705326) roll requested 2024-12-07T00:51:45,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705332 2024-12-07T00:51:45,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705332 2024-12-07T00:51:45,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1332, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705332) roll requested 2024-12-07T00:51:45,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705338 2024-12-07T00:51:45,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705338 2024-12-07T00:51:45,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1333, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705338) roll requested 2024-12-07T00:51:45,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705344 2024-12-07T00:51:45,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705344 2024-12-07T00:51:45,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1334, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705344) roll requested 2024-12-07T00:51:45,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705350 2024-12-07T00:51:45,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705350 2024-12-07T00:51:45,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1335, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705350) roll requested 2024-12-07T00:51:45,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705356 2024-12-07T00:51:45,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705356 2024-12-07T00:51:45,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1336, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705356) roll requested 2024-12-07T00:51:45,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705363 2024-12-07T00:51:45,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705363 2024-12-07T00:51:45,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1337, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705363) roll requested 2024-12-07T00:51:45,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705368 2024-12-07T00:51:45,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705368 2024-12-07T00:51:45,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1338, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705368) roll requested 2024-12-07T00:51:45,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705374 2024-12-07T00:51:45,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705374 2024-12-07T00:51:45,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1339, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705374) roll requested 2024-12-07T00:51:45,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705380 2024-12-07T00:51:45,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705380 2024-12-07T00:51:45,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1340, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705380) roll requested 2024-12-07T00:51:45,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705386 2024-12-07T00:51:45,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705386 2024-12-07T00:51:45,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1341, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705386) roll requested 2024-12-07T00:51:45,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705392 2024-12-07T00:51:45,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705392 2024-12-07T00:51:45,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1342, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705392) roll requested 2024-12-07T00:51:45,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705398 2024-12-07T00:51:45,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705398 2024-12-07T00:51:45,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1343, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705398) roll requested 2024-12-07T00:51:45,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705404 2024-12-07T00:51:45,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705404 2024-12-07T00:51:45,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1344, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705404) roll requested 2024-12-07T00:51:45,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705409 2024-12-07T00:51:45,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705409 2024-12-07T00:51:45,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1345, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705409) roll requested 2024-12-07T00:51:45,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705415 2024-12-07T00:51:45,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705415 2024-12-07T00:51:45,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1346, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705415) roll requested 2024-12-07T00:51:45,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705421 2024-12-07T00:51:45,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705421 2024-12-07T00:51:45,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1347, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705421) roll requested 2024-12-07T00:51:45,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705427 2024-12-07T00:51:45,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705427 2024-12-07T00:51:45,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1348, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705427) roll requested 2024-12-07T00:51:45,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705433 2024-12-07T00:51:45,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705433 2024-12-07T00:51:45,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1349, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705433) roll requested 2024-12-07T00:51:45,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705439 2024-12-07T00:51:45,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705439 2024-12-07T00:51:45,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1350, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705439) roll requested 2024-12-07T00:51:45,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705445 2024-12-07T00:51:45,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705445 2024-12-07T00:51:45,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1351, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705445) roll requested 2024-12-07T00:51:45,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705451 2024-12-07T00:51:45,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705451 2024-12-07T00:51:45,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1352, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705451) roll requested 2024-12-07T00:51:45,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705457 2024-12-07T00:51:45,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705457 2024-12-07T00:51:45,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1353, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705457) roll requested 2024-12-07T00:51:45,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705463 2024-12-07T00:51:45,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705463 2024-12-07T00:51:45,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1354, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705463) roll requested 2024-12-07T00:51:45,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705469 2024-12-07T00:51:45,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705469 2024-12-07T00:51:45,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1355, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705469) roll requested 2024-12-07T00:51:45,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705475 2024-12-07T00:51:45,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705475 2024-12-07T00:51:45,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1356, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705475) roll requested 2024-12-07T00:51:45,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705482 2024-12-07T00:51:45,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705482 2024-12-07T00:51:45,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1357, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705482) roll requested 2024-12-07T00:51:45,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705488 2024-12-07T00:51:45,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705488 2024-12-07T00:51:45,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1358, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705488) roll requested 2024-12-07T00:51:45,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705493 2024-12-07T00:51:45,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705493 2024-12-07T00:51:45,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1359, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705493) roll requested 2024-12-07T00:51:45,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705499 2024-12-07T00:51:45,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705499 2024-12-07T00:51:45,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1360, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705499) roll requested 2024-12-07T00:51:45,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705506 2024-12-07T00:51:45,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705506 2024-12-07T00:51:45,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1361, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705506) roll requested 2024-12-07T00:51:45,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705513 2024-12-07T00:51:45,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705513 2024-12-07T00:51:45,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1362, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705513) roll requested 2024-12-07T00:51:45,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705519 2024-12-07T00:51:45,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705519 2024-12-07T00:51:45,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1363, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705519) roll requested 2024-12-07T00:51:45,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705525 2024-12-07T00:51:45,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705525 2024-12-07T00:51:45,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1364, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705525) roll requested 2024-12-07T00:51:45,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705531 2024-12-07T00:51:45,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705531 2024-12-07T00:51:45,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1365, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705531) roll requested 2024-12-07T00:51:45,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705537 2024-12-07T00:51:45,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705537 2024-12-07T00:51:45,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1366, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705537) roll requested 2024-12-07T00:51:45,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705543 2024-12-07T00:51:45,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705543 2024-12-07T00:51:45,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1367, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705543) roll requested 2024-12-07T00:51:45,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705549 2024-12-07T00:51:45,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705549 2024-12-07T00:51:45,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1368, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705549) roll requested 2024-12-07T00:51:45,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705555 2024-12-07T00:51:45,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705555 2024-12-07T00:51:45,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1369, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705555) roll requested 2024-12-07T00:51:45,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705561 2024-12-07T00:51:45,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705561 2024-12-07T00:51:45,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1370, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705561) roll requested 2024-12-07T00:51:45,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705567 2024-12-07T00:51:45,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705567 2024-12-07T00:51:45,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1371, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705567) roll requested 2024-12-07T00:51:45,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705573 2024-12-07T00:51:45,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705573 2024-12-07T00:51:45,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1372, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705573) roll requested 2024-12-07T00:51:45,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705579 2024-12-07T00:51:45,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705579 2024-12-07T00:51:45,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1373, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705579) roll requested 2024-12-07T00:51:45,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705586 2024-12-07T00:51:45,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705586 2024-12-07T00:51:45,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1374, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705586) roll requested 2024-12-07T00:51:45,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705592 2024-12-07T00:51:45,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705592 2024-12-07T00:51:45,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1375, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705592) roll requested 2024-12-07T00:51:45,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705598 2024-12-07T00:51:45,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705598 2024-12-07T00:51:45,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1376, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705598) roll requested 2024-12-07T00:51:45,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705604 2024-12-07T00:51:45,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705604 2024-12-07T00:51:45,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1377, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705604) roll requested 2024-12-07T00:51:45,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705610 2024-12-07T00:51:45,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705610 2024-12-07T00:51:45,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1378, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705610) roll requested 2024-12-07T00:51:45,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705616 2024-12-07T00:51:45,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705616 2024-12-07T00:51:45,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1379, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705616) roll requested 2024-12-07T00:51:45,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705622 2024-12-07T00:51:45,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705622 2024-12-07T00:51:45,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1380, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705622) roll requested 2024-12-07T00:51:45,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705627 2024-12-07T00:51:45,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705627 2024-12-07T00:51:45,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1381, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705627) roll requested 2024-12-07T00:51:45,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705633 2024-12-07T00:51:45,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705633 2024-12-07T00:51:45,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1382, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705633) roll requested 2024-12-07T00:51:45,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705639 2024-12-07T00:51:45,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705639 2024-12-07T00:51:45,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1383, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705639) roll requested 2024-12-07T00:51:45,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705645 2024-12-07T00:51:45,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705645 2024-12-07T00:51:45,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1384, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705645) roll requested 2024-12-07T00:51:45,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705651 2024-12-07T00:51:45,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705651 2024-12-07T00:51:45,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1385, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705651) roll requested 2024-12-07T00:51:45,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705657 2024-12-07T00:51:45,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705657 2024-12-07T00:51:45,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1386, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705657) roll requested 2024-12-07T00:51:45,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705663 2024-12-07T00:51:45,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705663 2024-12-07T00:51:45,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1387, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705663) roll requested 2024-12-07T00:51:45,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705668 2024-12-07T00:51:45,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705668 2024-12-07T00:51:45,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1388, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705668) roll requested 2024-12-07T00:51:45,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705674 2024-12-07T00:51:45,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705674 2024-12-07T00:51:45,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1389, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705674) roll requested 2024-12-07T00:51:45,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705679 2024-12-07T00:51:45,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705679 2024-12-07T00:51:45,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1390, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705679) roll requested 2024-12-07T00:51:45,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705685 2024-12-07T00:51:45,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705685 2024-12-07T00:51:45,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1391, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705685) roll requested 2024-12-07T00:51:45,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705691 2024-12-07T00:51:45,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705691 2024-12-07T00:51:45,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1392, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705691) roll requested 2024-12-07T00:51:45,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705697 2024-12-07T00:51:45,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705697 2024-12-07T00:51:45,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1393, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705697) roll requested 2024-12-07T00:51:45,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705703 2024-12-07T00:51:45,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705703 2024-12-07T00:51:45,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1394, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705703) roll requested 2024-12-07T00:51:45,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705709 2024-12-07T00:51:45,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705709 2024-12-07T00:51:45,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1395, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705709) roll requested 2024-12-07T00:51:45,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705715 2024-12-07T00:51:45,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705715 2024-12-07T00:51:45,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1396, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705715) roll requested 2024-12-07T00:51:45,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705721 2024-12-07T00:51:45,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705721 2024-12-07T00:51:45,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1397, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705721) roll requested 2024-12-07T00:51:45,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705727 2024-12-07T00:51:45,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705727 2024-12-07T00:51:45,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1398, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705727) roll requested 2024-12-07T00:51:45,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705732 2024-12-07T00:51:45,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705732 2024-12-07T00:51:45,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1399, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705732) roll requested 2024-12-07T00:51:45,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705739 2024-12-07T00:51:45,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705739 2024-12-07T00:51:45,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1400, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705739) roll requested 2024-12-07T00:51:45,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705744 2024-12-07T00:51:45,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705744 2024-12-07T00:51:45,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1401, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705744) roll requested 2024-12-07T00:51:45,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705750 2024-12-07T00:51:45,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705750 2024-12-07T00:51:45,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1402, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705750) roll requested 2024-12-07T00:51:45,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705756 2024-12-07T00:51:45,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705756 2024-12-07T00:51:45,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1403, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705756) roll requested 2024-12-07T00:51:45,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705762 2024-12-07T00:51:45,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705762 2024-12-07T00:51:45,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1404, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705762) roll requested 2024-12-07T00:51:45,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705768 2024-12-07T00:51:45,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705768 2024-12-07T00:51:45,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1405, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705768) roll requested 2024-12-07T00:51:45,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705773 2024-12-07T00:51:45,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705773 2024-12-07T00:51:45,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1406, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705773) roll requested 2024-12-07T00:51:45,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705779 2024-12-07T00:51:45,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705779 2024-12-07T00:51:45,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1407, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705779) roll requested 2024-12-07T00:51:45,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705786 2024-12-07T00:51:45,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705786 2024-12-07T00:51:45,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1408, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705786) roll requested 2024-12-07T00:51:45,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705792 2024-12-07T00:51:45,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705792 2024-12-07T00:51:45,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1409, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705792) roll requested 2024-12-07T00:51:45,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705798 2024-12-07T00:51:45,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705798 2024-12-07T00:51:45,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1410, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705798) roll requested 2024-12-07T00:51:45,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705805 2024-12-07T00:51:45,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705805 2024-12-07T00:51:45,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1411, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705805) roll requested 2024-12-07T00:51:45,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705812 2024-12-07T00:51:45,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705812 2024-12-07T00:51:45,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1412, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705812) roll requested 2024-12-07T00:51:45,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705818 2024-12-07T00:51:45,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705818 2024-12-07T00:51:45,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1413, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705818) roll requested 2024-12-07T00:51:45,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705825 2024-12-07T00:51:45,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705825 2024-12-07T00:51:45,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1414, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705825) roll requested 2024-12-07T00:51:45,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705831 2024-12-07T00:51:45,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705831 2024-12-07T00:51:45,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1415, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705831) roll requested 2024-12-07T00:51:45,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705837 2024-12-07T00:51:45,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705837 2024-12-07T00:51:45,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1416, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705837) roll requested 2024-12-07T00:51:45,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705843 2024-12-07T00:51:45,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705843 2024-12-07T00:51:45,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1417, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705843) roll requested 2024-12-07T00:51:45,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705848 2024-12-07T00:51:45,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705848 2024-12-07T00:51:45,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1418, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705848) roll requested 2024-12-07T00:51:45,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705854 2024-12-07T00:51:45,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705854 2024-12-07T00:51:45,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1419, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705854) roll requested 2024-12-07T00:51:45,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705860 2024-12-07T00:51:45,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705860 2024-12-07T00:51:45,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1420, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705860) roll requested 2024-12-07T00:51:45,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705868 2024-12-07T00:51:45,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705868 2024-12-07T00:51:45,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1421, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705868) roll requested 2024-12-07T00:51:45,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705875 2024-12-07T00:51:45,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705875 2024-12-07T00:51:45,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1422, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705875) roll requested 2024-12-07T00:51:45,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705885 2024-12-07T00:51:45,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705885 2024-12-07T00:51:45,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1423, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705885) roll requested 2024-12-07T00:51:45,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705893 2024-12-07T00:51:45,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705893 2024-12-07T00:51:45,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1424, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705893) roll requested 2024-12-07T00:51:45,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705902 2024-12-07T00:51:45,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705902 2024-12-07T00:51:45,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1425, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705902) roll requested 2024-12-07T00:51:45,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705911 2024-12-07T00:51:45,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705911 2024-12-07T00:51:45,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1426, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705911) roll requested 2024-12-07T00:51:45,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705920 2024-12-07T00:51:45,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705920 2024-12-07T00:51:45,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1427, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705920) roll requested 2024-12-07T00:51:45,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705927 2024-12-07T00:51:45,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705927 2024-12-07T00:51:45,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1428, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705927) roll requested 2024-12-07T00:51:45,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705933 2024-12-07T00:51:45,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705933 2024-12-07T00:51:45,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1429, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705933) roll requested 2024-12-07T00:51:45,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705942 2024-12-07T00:51:45,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705942 2024-12-07T00:51:45,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1430, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705942) roll requested 2024-12-07T00:51:45,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705951 2024-12-07T00:51:45,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705951 2024-12-07T00:51:45,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1431, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705951) roll requested 2024-12-07T00:51:45,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705961 2024-12-07T00:51:45,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705961 2024-12-07T00:51:45,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1432, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705961) roll requested 2024-12-07T00:51:45,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705969 2024-12-07T00:51:45,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705969 2024-12-07T00:51:45,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1433, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705969) roll requested 2024-12-07T00:51:45,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705976 2024-12-07T00:51:45,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705976 2024-12-07T00:51:45,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1434, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705976) roll requested 2024-12-07T00:51:45,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705984 2024-12-07T00:51:45,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705984 2024-12-07T00:51:45,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:45,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:45,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1435, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:45,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:45,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705984) roll requested 2024-12-07T00:51:45,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532705992 2024-12-07T00:51:45,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:45,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705992 2024-12-07T00:51:46,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1436, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532705992) roll requested 2024-12-07T00:51:46,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706001 2024-12-07T00:51:46,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532705992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706001 2024-12-07T00:51:46,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1437, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706001) roll requested 2024-12-07T00:51:46,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706010 2024-12-07T00:51:46,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706010 2024-12-07T00:51:46,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1438, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706010) roll requested 2024-12-07T00:51:46,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706019 2024-12-07T00:51:46,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706019 2024-12-07T00:51:46,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1439, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706019) roll requested 2024-12-07T00:51:46,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706028 2024-12-07T00:51:46,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706028 2024-12-07T00:51:46,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1440, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706028) roll requested 2024-12-07T00:51:46,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706037 2024-12-07T00:51:46,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706037 2024-12-07T00:51:46,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1441, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706037) roll requested 2024-12-07T00:51:46,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706048 2024-12-07T00:51:46,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706048 2024-12-07T00:51:46,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1442, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706048) roll requested 2024-12-07T00:51:46,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706058 2024-12-07T00:51:46,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706058 2024-12-07T00:51:46,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1443, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706058) roll requested 2024-12-07T00:51:46,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706067 2024-12-07T00:51:46,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706067 2024-12-07T00:51:46,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1444, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706067) roll requested 2024-12-07T00:51:46,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706079 2024-12-07T00:51:46,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706079 2024-12-07T00:51:46,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1445, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706079) roll requested 2024-12-07T00:51:46,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706087 2024-12-07T00:51:46,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706087 2024-12-07T00:51:46,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1446, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706087) roll requested 2024-12-07T00:51:46,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706095 2024-12-07T00:51:46,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706095 2024-12-07T00:51:46,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1447, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706095) roll requested 2024-12-07T00:51:46,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706103 2024-12-07T00:51:46,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706103 2024-12-07T00:51:46,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1448, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706103) roll requested 2024-12-07T00:51:46,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706113 2024-12-07T00:51:46,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706113 2024-12-07T00:51:46,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1449, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706113) roll requested 2024-12-07T00:51:46,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706119 2024-12-07T00:51:46,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706119 2024-12-07T00:51:46,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1450, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706119) roll requested 2024-12-07T00:51:46,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706126 2024-12-07T00:51:46,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706126 2024-12-07T00:51:46,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1451, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706126) roll requested 2024-12-07T00:51:46,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706134 2024-12-07T00:51:46,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706134 2024-12-07T00:51:46,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1452, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706134) roll requested 2024-12-07T00:51:46,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706142 2024-12-07T00:51:46,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706142 2024-12-07T00:51:46,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1453, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706142) roll requested 2024-12-07T00:51:46,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706150 2024-12-07T00:51:46,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706150 2024-12-07T00:51:46,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1454, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706150) roll requested 2024-12-07T00:51:46,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706156 2024-12-07T00:51:46,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706156 2024-12-07T00:51:46,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1455, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706156) roll requested 2024-12-07T00:51:46,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706163 2024-12-07T00:51:46,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706163 2024-12-07T00:51:46,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1456, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706163) roll requested 2024-12-07T00:51:46,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706171 2024-12-07T00:51:46,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706171 2024-12-07T00:51:46,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1457, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706171) roll requested 2024-12-07T00:51:46,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706179 2024-12-07T00:51:46,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706179 2024-12-07T00:51:46,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1458, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706179) roll requested 2024-12-07T00:51:46,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706189 2024-12-07T00:51:46,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706189 2024-12-07T00:51:46,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1459, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706189) roll requested 2024-12-07T00:51:46,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706198 2024-12-07T00:51:46,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706198 2024-12-07T00:51:46,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1460, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706198) roll requested 2024-12-07T00:51:46,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706208 2024-12-07T00:51:46,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706208 2024-12-07T00:51:46,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1461, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706208) roll requested 2024-12-07T00:51:46,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706217 2024-12-07T00:51:46,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706217 2024-12-07T00:51:46,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1462, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706217) roll requested 2024-12-07T00:51:46,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706226 2024-12-07T00:51:46,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706226 2024-12-07T00:51:46,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1463, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706226) roll requested 2024-12-07T00:51:46,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706235 2024-12-07T00:51:46,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706235 2024-12-07T00:51:46,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1464, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706235) roll requested 2024-12-07T00:51:46,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706243 2024-12-07T00:51:46,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706243 2024-12-07T00:51:46,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1465, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706243) roll requested 2024-12-07T00:51:46,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706253 2024-12-07T00:51:46,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706253 2024-12-07T00:51:46,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1466, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706253) roll requested 2024-12-07T00:51:46,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706260 2024-12-07T00:51:46,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706260 2024-12-07T00:51:46,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1467, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706260) roll requested 2024-12-07T00:51:46,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706266 2024-12-07T00:51:46,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706266 2024-12-07T00:51:46,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1468, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706266) roll requested 2024-12-07T00:51:46,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706273 2024-12-07T00:51:46,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706273 2024-12-07T00:51:46,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1469, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706273) roll requested 2024-12-07T00:51:46,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706280 2024-12-07T00:51:46,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706280 2024-12-07T00:51:46,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-12-07T00:51:46,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1470, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1733532706280) roll requested 2024-12-07T00:51:46,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733532706287 2024-12-07T00:51:46,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1733532706287 2024-12-07T00:51:46,295 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T00:51:46,298 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f692feca4f5b23e8ac2ba4a607b7670, disabling compactions & flushes 2024-12-07T00:51:46,298 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670. 2024-12-07T00:51:46,298 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670. 2024-12-07T00:51:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670. after waiting 0 ms 2024-12-07T00:51:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670. 2024-12-07T00:51:46,301 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670. 2024-12-07T00:51:46,301 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1733532675976.1f692feca4f5b23e8ac2ba4a607b7670. 2024-12-07T00:51:46,302 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f692feca4f5b23e8ac2ba4a607b7670: Waiting for close lock at 1733532706298Disabling compacts and flushes for region at 1733532706298Disabling writes for close at 1733532706299 (+1 ms)Writing region close event to WAL at 1733532706301 (+2 ms)Closed at 1733532706301 2024-12-07T00:51:46,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-12-07T00:51:46,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1471, max=55; forcing (partial) flush of 1 region(s): 1f692feca4f5b23e8ac2ba4a607b7670[MyCF] 2024-12-07T00:51:46,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 1f692feca4f5b23e8ac2ba4a607b7670, because it is not online on us 2024-12-07T00:51:46,303 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T00:51:46,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T00:51:46,646 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1733532706287) 2024-12-07T00:51:46,647 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5b9b4104-5d79-eeaa-a1ce-b0000b5329e1 2024-12-07T00:51:47,222 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=7543 (was 8323)